From cc7425e83d6e92f0066bddc3f9c3d0c104228137 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Tue, 13 Feb 2024 09:41:55 -0800 Subject: [PATCH 01/42] Fix a bunch of places we forget to aws_raise_error() (#1089) --- .github/workflows/ci.yml | 2 +- .github/workflows/codecov.yml | 2 +- include/aws/common/json.h | 2 +- source/json.c | 4 ++-- source/log_formatter.c | 2 +- source/logging.c | 4 ++-- tests/encoding_test.c | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c840938d..b25b05cff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: - 'main' env: - BUILDER_VERSION: v0.9.49 + BUILDER_VERSION: v0.9.55 BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net BUILDER_SOURCE: releases PACKAGE_NAME: aws-c-common diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 7a1adff09..317354e59 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -4,7 +4,7 @@ on: push: env: - BUILDER_VERSION: v0.9.37 + BUILDER_VERSION: v0.9.55 BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net BUILDER_SOURCE: releases PACKAGE_NAME: aws-c-common diff --git a/include/aws/common/json.h b/include/aws/common/json.h index b614387c0..b8c4e6cfe 100644 --- a/include/aws/common/json.h +++ b/include/aws/common/json.h @@ -400,7 +400,7 @@ int aws_byte_buf_append_json_string(const struct aws_json_value *value, struct a * @param value The aws_json_value to format. * @param output The destination for the JSON string * @return AWS_OP_SUCCESS if the JSON string was allocated to output without any errors - * Will return AWS_ERROR_INVALID_ARGUMENT if the value passed is not an aws_json_value or if there + * Will return AWS_OP_ERR if the value passed is not an aws_json_value or if there * aws an error appending the JSON into the byte buffer. */ AWS_COMMON_API diff --git a/source/json.c b/source/json.c index face2914c..326354d62 100644 --- a/source/json.c +++ b/source/json.c @@ -397,7 +397,7 @@ int aws_byte_buf_append_json_string(const struct aws_json_value *value, struct a char *tmp = cJSON_PrintUnformatted(cjson); if (tmp == NULL) { - return AWS_OP_ERR; + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } // Append the text to the byte buffer @@ -415,7 +415,7 @@ int aws_byte_buf_append_json_string_formatted(const struct aws_json_value *value char *tmp = cJSON_Print(cjson); if (tmp == NULL) { - return AWS_OP_ERR; + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } // Append the text to the byte buffer diff --git a/source/log_formatter.c b/source/log_formatter.c index 321312981..b6e74e407 100644 --- a/source/log_formatter.c +++ b/source/log_formatter.c @@ -205,7 +205,7 @@ static int s_default_aws_log_formatter_format( struct aws_default_log_formatter_impl *impl = formatter->impl; if (formatted_output == NULL) { - return AWS_OP_ERR; + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } /* diff --git a/source/logging.c b/source/logging.c index 9033a208c..46f5e7dbe 100644 --- a/source/logging.c +++ b/source/logging.c @@ -337,11 +337,11 @@ int aws_thread_id_t_to_string(aws_thread_id_t thread_id, char *buffer, size_t bu unsigned char c = bytes[i - 1]; int written = snprintf(buffer + current_index, bufsz - current_index, "%02x", c); if (written < 0) { - return AWS_OP_ERR; + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } current_index += written; if (bufsz <= current_index) { - return AWS_OP_ERR; + return aws_raise_error(AWS_ERROR_SHORT_BUFFER); } } return AWS_OP_SUCCESS; diff --git a/tests/encoding_test.c b/tests/encoding_test.c index d9863951a..23cd084d0 100644 --- a/tests/encoding_test.c +++ b/tests/encoding_test.c @@ -1298,7 +1298,7 @@ static int s_utf8_validation_callback_always_fails(const uint32_t codepoint, voi static int s_utf8_validation_callback_always_passes(const uint32_t codepoint, void *user_data) { (void)codepoint; (void)user_data; - return AWS_ERROR_SUCCESS; + return AWS_OP_SUCCESS; } static struct utf8_example s_valid_utf8_examples_for_callback[] = { From 15a25349d59852e2655c0920835644f2eb948d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Tue, 13 Feb 2024 18:09:29 +0000 Subject: [PATCH 02/42] Consistently use lowercase for Windows libraries and header files (#1058) Co-authored-by: Michael Graeb --- CMakeLists.txt | 4 ++-- cmake/AwsFeatureTests.cmake | 2 +- include/aws/common/condition_variable.h | 2 +- include/aws/common/mutex.h | 2 +- include/aws/common/rw_lock.h | 2 +- include/aws/testing/aws_test_harness.h | 2 +- source/allocator.c | 2 +- source/common.c | 2 +- source/windows/clock.c | 2 +- source/windows/condition_variable.c | 2 +- source/windows/device_random.c | 3 ++- source/windows/file.c | 2 +- source/windows/mutex.c | 2 +- source/windows/rw_lock.c | 3 ++- source/windows/thread.c | 2 +- 15 files changed, 18 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 47d8714f2..74f1ae059 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,7 +61,7 @@ file (GLOB AWS_COMMON_EXTERNAL_SRC option(AWS_NUM_CPU_CORES "Number of CPU cores of the target machine. Useful when cross-compiling." 0) if (WIN32) - set(WINDOWS_KERNEL_LIB "Kernel32" CACHE STRING "The name of the kernel library to link against (default: Kernel32)") + set(WINDOWS_KERNEL_LIB "kernel32" CACHE STRING "The name of the kernel library to link against (default: kernel32)") file(GLOB AWS_COMMON_OS_HEADERS "include/aws/common/windows/*" @@ -84,7 +84,7 @@ if (WIN32) # PSAPI_VERSION=1 is needed to support GetProcessMemoryInfo on both pre and # post Win7 OS's. list(APPEND PLATFORM_DEFINES PSAPI_VERSION=1) - list(APPEND PLATFORM_LIBS BCrypt ${WINDOWS_KERNEL_LIB} Ws2_32 Shlwapi Psapi) + list(APPEND PLATFORM_LIBS bcrypt ${WINDOWS_KERNEL_LIB} ws2_32 shlwapi psapi) else () file(GLOB AWS_COMMON_OS_HEADERS "include/aws/common/posix/*" diff --git a/cmake/AwsFeatureTests.cmake b/cmake/AwsFeatureTests.cmake index da27994ec..60a548cfd 100644 --- a/cmake/AwsFeatureTests.cmake +++ b/cmake/AwsFeatureTests.cmake @@ -35,7 +35,7 @@ if(NOT CMAKE_CROSSCOMPILING) endif() check_c_source_compiles(" - #include + #include #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) int main() { return 0; diff --git a/include/aws/common/condition_variable.h b/include/aws/common/condition_variable.h index 317dedb9c..ba5984cb8 100644 --- a/include/aws/common/condition_variable.h +++ b/include/aws/common/condition_variable.h @@ -33,7 +33,7 @@ struct aws_condition_variable { * You can do something like struct aws_condition_variable var = * AWS_CONDITION_VARIABLE_INIT; * - * If on Windows and you get an error about AWS_CONDITION_VARIABLE_INIT being undefined, please include Windows.h to get + * If on Windows and you get an error about AWS_CONDITION_VARIABLE_INIT being undefined, please include windows.h to get * CONDITION_VARIABLE_INIT. */ #ifdef _WIN32 diff --git a/include/aws/common/mutex.h b/include/aws/common/mutex.h index b64e5e32f..3a491abc9 100644 --- a/include/aws/common/mutex.h +++ b/include/aws/common/mutex.h @@ -8,7 +8,7 @@ #include #ifdef _WIN32 -/* NOTE: Do not use this macro before including Windows.h */ +/* NOTE: Do not use this macro before including windows.h */ # define AWSMUTEX_TO_WINDOWS(pMutex) (PSRWLOCK) & (pMutex)->mutex_handle #else # include diff --git a/include/aws/common/rw_lock.h b/include/aws/common/rw_lock.h index f8d86ae2a..538841f63 100644 --- a/include/aws/common/rw_lock.h +++ b/include/aws/common/rw_lock.h @@ -8,7 +8,7 @@ #include #ifdef _WIN32 -/* NOTE: Do not use this macro before including Windows.h */ +/* NOTE: Do not use this macro before including windows.h */ # define AWSSRW_TO_WINDOWS(pCV) (PSRWLOCK) pCV #else # include diff --git a/include/aws/testing/aws_test_harness.h b/include/aws/testing/aws_test_harness.h index 88927c2b0..d4d258b2c 100644 --- a/include/aws/testing/aws_test_harness.h +++ b/include/aws/testing/aws_test_harness.h @@ -513,7 +513,7 @@ static inline int s_aws_run_test_case(struct aws_test_harness *harness) { /* https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences */ #ifdef _WIN32 -# include +# include # ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING # define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 diff --git a/source/allocator.c b/source/allocator.c index 02847b1ae..d3d1e98bd 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -12,7 +12,7 @@ #include #ifdef _WIN32 -# include +# include #endif #ifdef __MACH__ diff --git a/source/common.c b/source/common.c index f734c16e6..2c971b5b4 100644 --- a/source/common.c +++ b/source/common.c @@ -14,7 +14,7 @@ #include #ifdef _WIN32 -# include +# include #else # include #endif diff --git a/source/windows/clock.c b/source/windows/clock.c index b7b0b3820..11a5abb62 100644 --- a/source/windows/clock.c +++ b/source/windows/clock.c @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0. */ -#include #include +#include static const uint64_t FILE_TIME_TO_NS = 100; static const uint64_t EC_TO_UNIX_EPOCH = 11644473600LL; diff --git a/source/windows/condition_variable.c b/source/windows/condition_variable.c index 5a989ead0..07d6a0cdd 100644 --- a/source/windows/condition_variable.c +++ b/source/windows/condition_variable.c @@ -8,7 +8,7 @@ #include #include -#include +#include #define AWSCV_TO_WINDOWS(pCV) (PCONDITION_VARIABLE) & (pCV)->condition_handle diff --git a/source/windows/device_random.c b/source/windows/device_random.c index 0233d6890..6cb92d43e 100644 --- a/source/windows/device_random.c +++ b/source/windows/device_random.c @@ -7,7 +7,8 @@ #include #include -#include +#include + #include static BCRYPT_ALG_HANDLE s_alg_handle = NULL; diff --git a/source/windows/file.c b/source/windows/file.c index 7d4bb96d6..b3dfdf92c 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -8,9 +8,9 @@ #include #include -#include #include #include +#include #include #include diff --git a/source/windows/mutex.c b/source/windows/mutex.c index aedefd2af..75e7a23d1 100644 --- a/source/windows/mutex.c +++ b/source/windows/mutex.c @@ -6,7 +6,7 @@ #include #include -#include +#include /* Convert a string from a macro to a wide string */ #define WIDEN2(s) L## #s diff --git a/source/windows/rw_lock.c b/source/windows/rw_lock.c index 36cc81aa9..72a6e4378 100644 --- a/source/windows/rw_lock.c +++ b/source/windows/rw_lock.c @@ -6,7 +6,8 @@ #include #include -#include +#include + #include /* Convert a string from a macro to a wide string */ diff --git a/source/windows/thread.c b/source/windows/thread.c index 9124afefb..b172178ce 100644 --- a/source/windows/thread.c +++ b/source/windows/thread.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include From fcadc0dd5d8a26134c8bbf08c58e30eff50d177b Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Tue, 5 Mar 2024 09:16:27 -0800 Subject: [PATCH 03/42] Accept all RFC3339-compliant timestamps (#1093) **Issue:** `aws_date_time` did not accept "2024-02-23 23:06:27+00:00", despite that being a valid RFC 3339 timestamp. **Background:** - [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601): is a standard for date and time, but has a lot of options in it. - `aws_date_time` has enums for ISO_8601, but does not claim to support every single option - One option it did support though, was Basic vs Extended format. - Basic: no separators between "YYYYMMDD" and "HHMMSS" - Extended: has separators between "YYYY-MM-DD" and "HH-MM-SS" - [RFC 3339](https://datatracker.ietf.org/doc/html/rfc3339): details a subset of ISO 8601 for use in internet timestamps - [Smithy 2.0](https://smithy.io/2.0/spec/protocol-traits.html#timestamp-formats) says that timestamps should use RFC 3339 - `aws_date_time` does not fully support RFC 3339. The unsupported features in the string above were: 1) Space instead of "T" between date and time 2) "+00:00" offset instead of "Z" after time - I personally found the [parsing code for ISO_8601](https://github.com/awslabs/aws-c-common/blob/15a25349d59852e2655c0920835644f2eb948d77/source/date_time.c#L326) hard to follow, and the [ISO_8601_BASIC code](https://github.com/awslabs/aws-c-common/blob/15a25349d59852e2655c0920835644f2eb948d77/source/date_time.c#L221) was a big copy/paste of that. **Description of changes:** - The code is rewritten to be simpler. It is no longer a state machine. - Combine parsing code for `ISO_8601` and `ISO_8601_BASIC` into 1 function. - The parse is lenient now, accepting Basic or Extended timestamps, regardless of whether the `ISO_8601` or `ISO_8601_BASIC` enum is passed in. - Our `ISO_8601` code already allowed separators to be omitted from time, which shows evidence that lenience is good. - This matches the leniency of Python's [dateutil.parser.isoparse(str)](https://dateutil.readthedocs.io/en/stable/parser.html) which allows "2024-02-23T23:06:27Z" or "20240223T230627Z" or "2024-02-23T230627Z" or "20240223T23:06:27Z" - Allow space instead of "T" - Allows lowercase "t" and "z" - Support offsets like "+12:34", instead of just "Z" - Allow ",123" for fractional seconds, not just ".123" - The Basic parse code had a bug where did not expect any characters between seconds and fractional-seconds. This is wrong. Python's dateutil rejects this. --- include/aws/common/date_time.h | 7 +- source/date_time.c | 424 +++++++++++++++------------------ tests/CMakeLists.txt | 3 +- tests/date_time_test.c | 113 ++++----- 4 files changed, 243 insertions(+), 304 deletions(-) diff --git a/include/aws/common/date_time.h b/include/aws/common/date_time.h index 7e13740bd..72ca09a86 100644 --- a/include/aws/common/date_time.h +++ b/include/aws/common/date_time.h @@ -80,11 +80,14 @@ AWS_COMMON_API void aws_date_time_init_epoch_secs(struct aws_date_time *dt, doub * Initializes dt to be the time represented by date_str in format 'fmt'. Returns AWS_OP_SUCCESS if the * string was successfully parsed, returns AWS_OP_ERR if parsing failed. * + * The parser is lenient regarding AWS_DATE_FORMAT_ISO_8601 vs AWS_DATE_FORMAT_ISO_8601_BASIC. + * Regardless of which you pass in, both "2002-10-02T08:05:09Z" and "20021002T080509Z" would be accepted. + * * Notes for AWS_DATE_FORMAT_RFC822: * If no time zone information is provided, it is assumed to be local time (please don't do this). * - * If the time zone is something other than something indicating Universal Time (e.g. Z, UT, UTC, or GMT) or an offset - * from UTC (e.g. +0100, -0700), parsing will fail. + * Only time zones indicating Universal Time (e.g. Z, UT, UTC, or GMT), + * or offsets from UTC (e.g. +0100, -0700), are accepted. * * Really, it's just better if you always use Universal Time. */ diff --git a/source/date_time.c b/source/date_time.c index 49793074c..0c1869bce 100644 --- a/source/date_time.c +++ b/source/date_time.c @@ -131,7 +131,7 @@ static bool is_utc_time_zone(const char *str) { size_t len = strlen(str); if (len > 0) { - if (str[0] == 'Z') { + if (tolower((uint8_t)str[0]) == 'z') { return true; } @@ -207,232 +207,7 @@ enum parser_state { FINISHED, }; -static int s_parse_iso_8601_basic(const struct aws_byte_cursor *date_str_cursor, struct tm *parsed_time) { - size_t index = 0; - size_t state_start_index = 0; - enum parser_state state = ON_YEAR; - bool error = false; - - AWS_ZERO_STRUCT(*parsed_time); - - while (state < FINISHED && !error && index < date_str_cursor->len) { - char c = (char)date_str_cursor->ptr[index]; - size_t sub_index = index - state_start_index; - switch (state) { - case ON_YEAR: - if (aws_isdigit(c)) { - parsed_time->tm_year = parsed_time->tm_year * 10 + (c - '0'); - if (sub_index == 3) { - state = ON_MONTH; - state_start_index = index + 1; - parsed_time->tm_year -= 1900; - } - } else { - error = true; - } - break; - - case ON_MONTH: - if (aws_isdigit(c)) { - parsed_time->tm_mon = parsed_time->tm_mon * 10 + (c - '0'); - if (sub_index == 1) { - state = ON_MONTH_DAY; - state_start_index = index + 1; - parsed_time->tm_mon -= 1; - } - } else { - error = true; - } - break; - - case ON_MONTH_DAY: - if (c == 'T' && sub_index == 2) { - state = ON_HOUR; - state_start_index = index + 1; - } else if (aws_isdigit(c)) { - parsed_time->tm_mday = parsed_time->tm_mday * 10 + (c - '0'); - } else { - error = true; - } - break; - - case ON_HOUR: - if (aws_isdigit(c)) { - parsed_time->tm_hour = parsed_time->tm_hour * 10 + (c - '0'); - if (sub_index == 1) { - state = ON_MINUTE; - state_start_index = index + 1; - } - } else { - error = true; - } - break; - - case ON_MINUTE: - if (aws_isdigit(c)) { - parsed_time->tm_min = parsed_time->tm_min * 10 + (c - '0'); - if (sub_index == 1) { - state = ON_SECOND; - state_start_index = index + 1; - } - } else { - error = true; - } - break; - - case ON_SECOND: - if (aws_isdigit(c)) { - parsed_time->tm_sec = parsed_time->tm_sec * 10 + (c - '0'); - if (sub_index == 1) { - state = ON_TZ; - state_start_index = index + 1; - } - } else { - error = true; - } - break; - - case ON_TZ: - if (c == 'Z' && (sub_index == 0 || sub_index == 3)) { - state = FINISHED; - } else if (!aws_isdigit(c) || sub_index > 3) { - error = true; - } - break; - - default: - error = true; - break; - } - - index++; - } - - /* ISO8601 supports date only with no time portion. state ==ON_MONTH_DAY catches this case. */ - return (state == FINISHED || state == ON_MONTH_DAY) && !error ? AWS_OP_SUCCESS : AWS_OP_ERR; -} - -static int s_parse_iso_8601(const struct aws_byte_cursor *date_str_cursor, struct tm *parsed_time) { - size_t index = 0; - size_t state_start_index = 0; - enum parser_state state = ON_YEAR; - bool error = false; - bool advance = true; - - AWS_ZERO_STRUCT(*parsed_time); - - while (state < FINISHED && !error && index < date_str_cursor->len) { - char c = (char)date_str_cursor->ptr[index]; - switch (state) { - case ON_YEAR: - if (c == '-' && index - state_start_index == 4) { - state = ON_MONTH; - state_start_index = index + 1; - parsed_time->tm_year -= 1900; - } else if (aws_isdigit(c)) { - parsed_time->tm_year = parsed_time->tm_year * 10 + (c - '0'); - } else { - error = true; - } - break; - case ON_MONTH: - if (c == '-' && index - state_start_index == 2) { - state = ON_MONTH_DAY; - state_start_index = index + 1; - parsed_time->tm_mon -= 1; - } else if (aws_isdigit(c)) { - parsed_time->tm_mon = parsed_time->tm_mon * 10 + (c - '0'); - } else { - error = true; - } - - break; - case ON_MONTH_DAY: - if (c == 'T' && index - state_start_index == 2) { - state = ON_HOUR; - state_start_index = index + 1; - } else if (aws_isdigit(c)) { - parsed_time->tm_mday = parsed_time->tm_mday * 10 + (c - '0'); - } else { - error = true; - } - break; - /* note: no time portion is spec compliant. */ - case ON_HOUR: - /* time parts can be delimited by ':' or just concatenated together, but must always be 2 digits. */ - if (index - state_start_index == 2) { - state = ON_MINUTE; - state_start_index = index + 1; - if (aws_isdigit(c)) { - state_start_index = index; - advance = false; - } else if (c != ':') { - error = true; - } - } else if (aws_isdigit(c)) { - parsed_time->tm_hour = parsed_time->tm_hour * 10 + (c - '0'); - } else { - error = true; - } - - break; - case ON_MINUTE: - /* time parts can be delimited by ':' or just concatenated together, but must always be 2 digits. */ - if (index - state_start_index == 2) { - state = ON_SECOND; - state_start_index = index + 1; - if (aws_isdigit(c)) { - state_start_index = index; - advance = false; - } else if (c != ':') { - error = true; - } - } else if (aws_isdigit(c)) { - parsed_time->tm_min = parsed_time->tm_min * 10 + (c - '0'); - } else { - error = true; - } - - break; - case ON_SECOND: - if (c == 'Z' && index - state_start_index == 2) { - state = FINISHED; - state_start_index = index + 1; - } else if (c == '.' && index - state_start_index == 2) { - state = ON_TZ; - state_start_index = index + 1; - } else if (aws_isdigit(c)) { - parsed_time->tm_sec = parsed_time->tm_sec * 10 + (c - '0'); - } else { - error = true; - } - - break; - case ON_TZ: - if (c == 'Z') { - state = FINISHED; - state_start_index = index + 1; - } else if (!aws_isdigit(c)) { - error = true; - } - break; - default: - error = true; - break; - } - - if (advance) { - index++; - } else { - advance = true; - } - } - - /* ISO8601 supports date only with no time portion. state ==ON_MONTH_DAY catches this case. */ - return (state == FINISHED || state == ON_MONTH_DAY) && !error ? AWS_OP_SUCCESS : AWS_OP_ERR; -} - -static int s_parse_rfc_822( +static bool s_parse_rfc_822( const struct aws_byte_cursor *date_str_cursor, struct tm *parsed_time, struct aws_date_time *dt) { @@ -564,7 +339,186 @@ static int s_parse_rfc_822( } } - return error || state != ON_TZ ? AWS_OP_ERR : AWS_OP_SUCCESS; + return error || state != ON_TZ ? false : true; +} + +/* Returns true if the next N characters are digits, advancing the string and getting their numeric value */ +static bool s_read_n_digits(struct aws_byte_cursor *str, size_t n, int *out_val) { + int val = 0; + if (str->len < n) { + return false; + } + + for (size_t i = 0; i < n; ++i) { + uint8_t c = str->ptr[i]; + if (aws_isdigit(c)) { + val = val * 10 + (c - '0'); + } else { + return false; + } + } + + aws_byte_cursor_advance(str, n); + *out_val = val; + return true; +} + +/* Returns true if there's 1 more character, advancing the string and getting the character's value. */ +static bool s_read_1_char(struct aws_byte_cursor *str, uint8_t *out_c) { + if (str->len == 0) { + return false; + } + + *out_c = str->ptr[0]; + aws_byte_cursor_advance(str, 1); + return true; +} + +/* Returns true (and advances str) if next character is c */ +static bool s_advance_if_next_char_is(struct aws_byte_cursor *str, uint8_t c) { + if (str->len == 0 || str->ptr[0] != c) { + return false; + } + + aws_byte_cursor_advance(str, 1); + return true; +} + +/* If the (optional) fractional seconds (".123" or ",123") are next, str is advanced. + * Returns false if there was an error */ +static bool s_skip_optional_fractional_seconds(struct aws_byte_cursor *str) { + if (str->len == 0) { + return true; + } + + uint8_t c = str->ptr[0]; + if (c != '.' && c != ',') { + return true; + } + + size_t num_digits = 0; + for (size_t i = 1; i < str->len; ++i) { + if (aws_isdigit(str->ptr[i])) { + ++num_digits; + } else { + break; + } + } + + if (num_digits == 0) { + return false; + } + + aws_byte_cursor_advance(str, 1 + num_digits); + return true; +} + +/* Parses ISO 8601, both extended and basic format are accepted. + * Returns true if successful. */ +static bool s_parse_iso_8601(struct aws_byte_cursor str, struct tm *parsed_time, time_t *seconds_offset) { + AWS_ZERO_STRUCT(*parsed_time); + *seconds_offset = 0; + uint8_t c = 0; + + /* read year */ + if (!s_read_n_digits(&str, 4, &parsed_time->tm_year)) { + return false; + } + parsed_time->tm_year -= 1900; + + /* be lenient, allow date with separator or not */ + bool has_date_separator = s_advance_if_next_char_is(&str, '-'); + + /* read month */ + if (!s_read_n_digits(&str, 2, &parsed_time->tm_mon)) { + return false; + } + parsed_time->tm_mon -= 1; + + if (has_date_separator) { + if (!s_read_1_char(&str, &c) || c != '-') { + return false; + } + } + + /* read month-day */ + if (!s_read_n_digits(&str, 2, &parsed_time->tm_mday)) { + return false; + } + + /* ISO8601 supports date only with no time portion */ + if (str.len == 0) { + return true; + } + + /* followed by T or space (allowed by rfc3339#section-5.6) */ + if (!s_read_1_char(&str, &c) || (tolower(c) != 't' && c != ' ')) { + return false; + } + + /* read hours */ + if (!s_read_n_digits(&str, 2, &parsed_time->tm_hour)) { + return false; + } + + /* be lenient, allow time with separator or not */ + bool has_time_separator = s_advance_if_next_char_is(&str, ':'); + + /* read minutes */ + if (!s_read_n_digits(&str, 2, &parsed_time->tm_min)) { + return false; + } + + if (has_time_separator) { + if (!s_read_1_char(&str, &c) || c != ':') { + return false; + } + } + + /* read seconds */ + if (!s_read_n_digits(&str, 2, &parsed_time->tm_sec)) { + return false; + } + + /* fractional seconds are optional (discard value since tm struct has no corresponding field) */ + if (!s_skip_optional_fractional_seconds(&str)) { + return false; + } + + /* read final Z, or (+/-) indicating there will be an offset */ + if (!s_read_1_char(&str, &c)) { + return false; + } + + if (tolower(c) == 'z') { + /* Success! */ + return true; + } + + if (c != '+' && c != '-') { + return false; + } + + bool negative_offset = c == '-'; + + /* read hours offset */ + int hours_offset = 0; + if (!s_read_n_digits(&str, 2, &hours_offset)) { + return false; + } + + /* be lenient, allow offset with separator or not */ + s_advance_if_next_char_is(&str, ':'); + + /* read minutes offset */ + int minutes_offset = 0; + if (!s_read_n_digits(&str, 2, &minutes_offset)) { + return false; + } + + /* Success! */ + *seconds_offset = (time_t)(hours_offset * 3600 + minutes_offset * 60) * (negative_offset ? -1 : 1); + return true; } int aws_date_time_init_from_str_cursor( @@ -579,22 +533,16 @@ int aws_date_time_init_from_str_cursor( bool successfully_parsed = false; time_t seconds_offset = 0; - if (fmt == AWS_DATE_FORMAT_ISO_8601 || fmt == AWS_DATE_FORMAT_AUTO_DETECT) { - if (!s_parse_iso_8601(date_str_cursor, &parsed_time)) { - dt->utc_assumed = true; - successfully_parsed = true; - } - } - - if (fmt == AWS_DATE_FORMAT_ISO_8601_BASIC || (fmt == AWS_DATE_FORMAT_AUTO_DETECT && !successfully_parsed)) { - if (!s_parse_iso_8601_basic(date_str_cursor, &parsed_time)) { + if (fmt == AWS_DATE_FORMAT_ISO_8601 || fmt == AWS_DATE_FORMAT_ISO_8601_BASIC || + fmt == AWS_DATE_FORMAT_AUTO_DETECT) { + if (s_parse_iso_8601(*date_str_cursor, &parsed_time, &seconds_offset)) { dt->utc_assumed = true; successfully_parsed = true; } } if (fmt == AWS_DATE_FORMAT_RFC822 || (fmt == AWS_DATE_FORMAT_AUTO_DETECT && !successfully_parsed)) { - if (!s_parse_rfc_822(date_str_cursor, &parsed_time, dt)) { + if (s_parse_rfc_822(date_str_cursor, &parsed_time, dt)) { successfully_parsed = true; if (dt->utc_assumed) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 331e0504c..391f98fac 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -360,13 +360,12 @@ add_test_case(rfc822_utc_dos_prevented) add_test_case(rfc822_invalid_format) add_test_case(rfc822_invalid_tz) add_test_case(rfc822_invalid_auto_format) -add_test_case(iso8601_utc_parsing) +add_test_case(iso8601_parsing) add_test_case(iso8601_basic_utc_parsing) add_test_case(iso8601_utc_parsing_auto_detect) add_test_case(iso8601_basic_utc_parsing_auto_detect) add_test_case(iso8601_date_only_parsing) add_test_case(iso8601_basic_date_only_parsing) -add_test_case(iso8601_utc_no_colon_parsing) add_test_case(iso8601_utc_dos_prevented) add_test_case(iso8601_invalid_format) add_test_case(iso8601_invalid_auto_format) diff --git a/tests/date_time_test.c b/tests/date_time_test.c index 8272a4fd5..69f0417f0 100644 --- a/tests/date_time_test.c +++ b/tests/date_time_test.c @@ -19,7 +19,7 @@ static int s_test_rfc822_utc_parsing_fn(struct aws_allocator *allocator, void *c "Wed, 02 Oct 2002 08:05:09 UTC", }; - for (size_t i = 0; i < 4; ++i) { + for (size_t i = 0; i < AWS_ARRAY_SIZE(valid_utc_dates); ++i) { struct aws_date_time date_time; const char *date_str = valid_utc_dates[i]; struct aws_byte_buf date_buf = aws_byte_buf_from_c_str(date_str); @@ -284,52 +284,73 @@ static int s_test_rfc822_invalid_auto_format_fn(struct aws_allocator *allocator, AWS_TEST_CASE(rfc822_invalid_auto_format, s_test_rfc822_invalid_auto_format_fn) -static int s_test_iso8601_utc_parsing_fn(struct aws_allocator *allocator, void *ctx) { +static int s_test_iso8601_parsing_fn(struct aws_allocator *allocator, void *ctx) { (void)allocator; (void)ctx; - struct aws_date_time date_time; - const char *date_str = "2002-10-02T08:05:09.000Z"; - struct aws_byte_buf date_buf = aws_byte_buf_from_c_str(date_str); + /* array of {"date", "description"} */ + const char *valid_dates[][2] = { + {"2002-10-02T08:05:09.000Z", "extended format"}, + {"2002-10-02t08:05:09.000z", "lowercase T and Z"}, + {"2002-10-02 08:05:09Z", "space instead of T"}, /*allowed per NOTE in RFC3339#section-5.6 */ + {"2002-10-02T08:05:09+00:00", "offset instead of Z"}, + {"2002-10-01T19:31:09-12:34", "western offset"}, + {"2002-10-02T20:39:09+12:34", "eastern offset"}, + {"2002-10-02T08:05:09.000+00:00", "fractional seconds and offset"}, + {"20021002T080509.000Z", "basic format"}, + {"20021002T203909+1234", "basic format with fractional seconds"}, + {"2002-10-02T080509.000Z", "weird mix of extended date but basic time"}, + {"20021002T08:05:09.000Z", "weird mix of basic date but extended time"}, + {"2002-10-02T20:39:09+1234", "weird mix of extended date and time but basic fractional seconds"}, + }; - ASSERT_SUCCESS(aws_date_time_init_from_str(&date_time, &date_buf, AWS_DATE_FORMAT_ISO_8601)); - ASSERT_INT_EQUALS(AWS_DATE_DAY_OF_WEEK_WEDNESDAY, aws_date_time_day_of_week(&date_time, false)); - ASSERT_UINT_EQUALS(2, aws_date_time_month_day(&date_time, false)); - ASSERT_UINT_EQUALS(AWS_DATE_MONTH_OCTOBER, aws_date_time_month(&date_time, false)); - ASSERT_UINT_EQUALS(2002, aws_date_time_year(&date_time, false)); - ASSERT_UINT_EQUALS(8, aws_date_time_hour(&date_time, false)); - ASSERT_UINT_EQUALS(5, aws_date_time_minute(&date_time, false)); - ASSERT_UINT_EQUALS(9, aws_date_time_second(&date_time, false)); + for (size_t i = 0; i < AWS_ARRAY_SIZE(valid_dates); ++i) { + const char *date_str = valid_dates[i][0]; + const char *description = valid_dates[i][1]; + printf("checking date[%zu] \"%s\" (%s)\n", i, date_str, description); - uint8_t date_output[AWS_DATE_TIME_STR_MAX_LEN]; - AWS_ZERO_ARRAY(date_output); - struct aws_byte_buf str_output = aws_byte_buf_from_array(date_output, sizeof(date_output)); - str_output.len = 0; - ASSERT_SUCCESS(aws_date_time_to_utc_time_str(&date_time, AWS_DATE_FORMAT_ISO_8601, &str_output)); + struct aws_date_time date_time; + struct aws_byte_buf date_buf = aws_byte_buf_from_c_str(date_str); - const char *expected_date_str = "2002-10-02T08:05:09Z"; - struct aws_byte_buf expected_date_buf = aws_byte_buf_from_c_str(expected_date_str); - ASSERT_BIN_ARRAYS_EQUALS(expected_date_buf.buffer, expected_date_buf.len, str_output.buffer, str_output.len); + ASSERT_SUCCESS(aws_date_time_init_from_str(&date_time, &date_buf, AWS_DATE_FORMAT_ISO_8601)); + ASSERT_INT_EQUALS(AWS_DATE_DAY_OF_WEEK_WEDNESDAY, aws_date_time_day_of_week(&date_time, false)); + ASSERT_UINT_EQUALS(2, aws_date_time_month_day(&date_time, false)); + ASSERT_UINT_EQUALS(AWS_DATE_MONTH_OCTOBER, aws_date_time_month(&date_time, false)); + ASSERT_UINT_EQUALS(2002, aws_date_time_year(&date_time, false)); + ASSERT_UINT_EQUALS(8, aws_date_time_hour(&date_time, false)); + ASSERT_UINT_EQUALS(5, aws_date_time_minute(&date_time, false)); + ASSERT_UINT_EQUALS(9, aws_date_time_second(&date_time, false)); - AWS_ZERO_ARRAY(date_output); - str_output.len = 0; - ASSERT_SUCCESS(aws_date_time_to_utc_time_short_str(&date_time, AWS_DATE_FORMAT_ISO_8601, &str_output)); + uint8_t date_output[AWS_DATE_TIME_STR_MAX_LEN]; + AWS_ZERO_ARRAY(date_output); + struct aws_byte_buf str_output = aws_byte_buf_from_array(date_output, sizeof(date_output)); + str_output.len = 0; + ASSERT_SUCCESS(aws_date_time_to_utc_time_str(&date_time, AWS_DATE_FORMAT_ISO_8601, &str_output)); - const char *expected_short_str = "2002-10-02"; - struct aws_byte_buf expected_short_buf = aws_byte_buf_from_c_str(expected_short_str); + const char *expected_date_str = "2002-10-02T08:05:09Z"; + struct aws_byte_buf expected_date_buf = aws_byte_buf_from_c_str(expected_date_str); + ASSERT_BIN_ARRAYS_EQUALS(expected_date_buf.buffer, expected_date_buf.len, str_output.buffer, str_output.len); - ASSERT_BIN_ARRAYS_EQUALS(expected_short_buf.buffer, expected_short_buf.len, str_output.buffer, str_output.len); + AWS_ZERO_ARRAY(date_output); + str_output.len = 0; + ASSERT_SUCCESS(aws_date_time_to_utc_time_short_str(&date_time, AWS_DATE_FORMAT_ISO_8601, &str_output)); + + const char *expected_short_str = "2002-10-02"; + struct aws_byte_buf expected_short_buf = aws_byte_buf_from_c_str(expected_short_str); + + ASSERT_BIN_ARRAYS_EQUALS(expected_short_buf.buffer, expected_short_buf.len, str_output.buffer, str_output.len); + } return AWS_OP_SUCCESS; } -AWS_TEST_CASE(iso8601_utc_parsing, s_test_iso8601_utc_parsing_fn) +AWS_TEST_CASE(iso8601_parsing, s_test_iso8601_parsing_fn) static int s_test_iso8601_basic_utc_parsing_fn(struct aws_allocator *allocator, void *ctx) { (void)allocator; (void)ctx; struct aws_date_time date_time; - const char *date_str = "20021002T080509000Z"; + const char *date_str = "20021002T080509.000Z"; struct aws_byte_buf date_buf = aws_byte_buf_from_c_str(date_str); ASSERT_SUCCESS(aws_date_time_init_from_str(&date_time, &date_buf, AWS_DATE_FORMAT_ISO_8601_BASIC)); @@ -401,7 +422,7 @@ static int s_test_iso8601_basic_utc_parsing_auto_detect_fn(struct aws_allocator (void)ctx; struct aws_date_time date_time; - const char *date_str = "20021002T080509000Z"; + const char *date_str = "20021002T080509,000Z"; struct aws_byte_buf date_buf = aws_byte_buf_from_c_str(date_str); ASSERT_SUCCESS(aws_date_time_init_from_str(&date_time, &date_buf, AWS_DATE_FORMAT_AUTO_DETECT)); @@ -428,38 +449,6 @@ static int s_test_iso8601_basic_utc_parsing_auto_detect_fn(struct aws_allocator AWS_TEST_CASE(iso8601_basic_utc_parsing_auto_detect, s_test_iso8601_basic_utc_parsing_auto_detect_fn) -static int s_test_iso8601_utc_no_colon_parsing_fn(struct aws_allocator *allocator, void *ctx) { - (void)allocator; - (void)ctx; - - struct aws_date_time date_time; - const char *date_str = "2002-10-02T080509.000Z"; - struct aws_byte_buf date_buf = aws_byte_buf_from_c_str(date_str); - - ASSERT_SUCCESS(aws_date_time_init_from_str(&date_time, &date_buf, AWS_DATE_FORMAT_ISO_8601)); - ASSERT_INT_EQUALS(AWS_DATE_DAY_OF_WEEK_WEDNESDAY, aws_date_time_day_of_week(&date_time, false)); - ASSERT_UINT_EQUALS(2, aws_date_time_month_day(&date_time, false)); - ASSERT_UINT_EQUALS(AWS_DATE_MONTH_OCTOBER, aws_date_time_month(&date_time, false)); - ASSERT_UINT_EQUALS(2002, aws_date_time_year(&date_time, false)); - ASSERT_UINT_EQUALS(8, aws_date_time_hour(&date_time, false)); - ASSERT_UINT_EQUALS(5, aws_date_time_minute(&date_time, false)); - ASSERT_UINT_EQUALS(9, aws_date_time_second(&date_time, false)); - - uint8_t date_output[AWS_DATE_TIME_STR_MAX_LEN]; - AWS_ZERO_ARRAY(date_output); - struct aws_byte_buf str_output = aws_byte_buf_from_array(date_output, sizeof(date_output)); - str_output.len = 0; - ASSERT_SUCCESS(aws_date_time_to_utc_time_str(&date_time, AWS_DATE_FORMAT_ISO_8601, &str_output)); - - const char *expected_date_str = "2002-10-02T08:05:09Z"; - struct aws_byte_buf expected_date_buf = aws_byte_buf_from_c_str(expected_date_str); - ASSERT_BIN_ARRAYS_EQUALS(expected_date_buf.buffer, expected_date_buf.len, str_output.buffer, str_output.len); - - return AWS_OP_SUCCESS; -} - -AWS_TEST_CASE(iso8601_utc_no_colon_parsing, s_test_iso8601_utc_no_colon_parsing_fn) - static int s_test_iso8601_date_only_parsing_fn(struct aws_allocator *allocator, void *ctx) { (void)allocator; (void)ctx; From 4e1cfca4a8e3266e23c523aa2046dbc448a51b1b Mon Sep 17 00:00:00 2001 From: Aaron Todd Date: Fri, 8 Mar 2024 15:45:02 -0500 Subject: [PATCH 04/42] claim range for aws-crt-kotlin (#1095) and increase max package slots to 32 --- README.md | 2 +- include/aws/common/package.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3aaa9498e..95ab31533 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ have pre-slotted log subjects & error codes for each library. The currently allo | [0x3400, 0x3800) | aws-c-iot | | [0x3800, 0x3C00) | aws-c-s3 | | [0x3C00, 0x4000) | aws-c-sdkutils | -| [0x4000, 0x4400) | (reserved for future project) | +| [0x4000, 0x4400) | aws-crt-kotlin | | [0x4400, 0x4800) | (reserved for future project) | Each library should begin its error and log subject values at the beginning of its range and follow in sequence (don't skip codes). Upon diff --git a/include/aws/common/package.h b/include/aws/common/package.h index 0d7b3c2a5..c93fd5109 100644 --- a/include/aws/common/package.h +++ b/include/aws/common/package.h @@ -10,7 +10,7 @@ * Preliminary cap on the number of possible aws-c-libraries participating in shared enum ranges for * errors, log subjects, and other cross-library enums. Expandable as needed */ -#define AWS_PACKAGE_SLOTS 16 +#define AWS_PACKAGE_SLOTS 32 /* * Each aws-c-* and aws-crt-* library has a unique package id starting from zero. These are used to macro-calculate From 42119c5a1c191f397e3ad770cd900104e050ee34 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 11 Mar 2024 16:40:22 -0700 Subject: [PATCH 05/42] Remove unused external headers (#1097) --- CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 74f1ae059..7e5c5cd9f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,9 +36,6 @@ file(GLOB AWS_COMMON_HEADERS "include/aws/common/*.inl" ) -file (GLOB AWS_COMMON_EXTERNAL_HEADERS - "include/aws/common/external/*.h") - file (GLOB AWS_COMMON_EXTERNAL_INSTALLED_HEADERS "include/aws/common/external/ittnotify.h") @@ -82,7 +79,7 @@ if (WIN32) list(APPEND PLATFORM_DEFINES WINDOWS_KERNEL_LIB=${WINDOWS_KERNEL_LIB}) # PSAPI_VERSION=1 is needed to support GetProcessMemoryInfo on both pre and - # post Win7 OS's. + # post Win7 OS's. list(APPEND PLATFORM_DEFINES PSAPI_VERSION=1) list(APPEND PLATFORM_LIBS bcrypt ${WINDOWS_KERNEL_LIB} ws2_32 shlwapi psapi) else () @@ -173,7 +170,6 @@ file(GLOB COMMON_HEADERS ${AWS_COMMON_HEADERS} ${AWS_COMMON_OS_HEADERS} ${AWS_COMMON_PRIV_HEADERS} - ${AWS_COMMON_EXTERNAL_HEADERS} ${AWS_TEST_HEADERS} ) From 71c6ab7d68527767f042215fc7aa4b17d3774c13 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Fri, 15 Mar 2024 12:33:45 -0700 Subject: [PATCH 06/42] Best Effort Thread Pinning to CPU_ID (#1096) --- source/posix/thread.c | 19 ++++++++++++++++--- tests/CMakeLists.txt | 1 + tests/thread_test.c | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/source/posix/thread.c b/source/posix/thread.c index 0b32bebdd..af7fac84c 100644 --- a/source/posix/thread.c +++ b/source/posix/thread.c @@ -296,9 +296,9 @@ int aws_thread_launch( attr_return = pthread_attr_setaffinity_np(attributes_ptr, sizeof(cpuset), &cpuset); if (attr_return) { - AWS_LOGF_ERROR( + AWS_LOGF_WARN( AWS_LS_COMMON_THREAD, - "id=%p: pthread_attr_setaffinity_np() failed with %d.", + "id=%p: pthread_attr_setaffinity_np() failed with %d. Continuing without cpu affinity", (void *)thread, attr_return); goto cleanup; @@ -382,7 +382,20 @@ int aws_thread_launch( if (attr_return) { s_thread_wrapper_destroy(wrapper); - + if (options && options->cpu_id >= 0) { + /* + * `pthread_create` can fail with an `EINVAL` error or `EDEADLK` on freebasd if the `cpu_id` is + * restricted/invalid. Since the pinning to a particular `cpu_id` is supposed to be best-effort, try to + * launch a thread again without pinning to a specific cpu_id. + */ + AWS_LOGF_INFO( + AWS_LS_COMMON_THREAD, + "id=%p: Attempting to launch the thread again without pinning to a cpu_id", + (void *)thread); + struct aws_thread_options new_options = *options; + new_options.cpu_id = -1; + return aws_thread_launch(thread, func, arg, &new_options); + } switch (attr_return) { case EINVAL: return aws_raise_error(AWS_ERROR_THREAD_INVALID_SETTINGS); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 391f98fac..d2462469a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -40,6 +40,7 @@ add_test_case(aws_load_error_strings_test) add_test_case(aws_assume_compiles_test) add_test_case(thread_creation_join_test) +add_test_case(thread_creation_join_invalid_cpu_id_test) add_test_case(thread_atexit_test) add_test_case(test_managed_thread_join) add_test_case(test_managed_thread_join_timeout) diff --git a/tests/thread_test.c b/tests/thread_test.c index 4cf3278b9..50085d843 100644 --- a/tests/thread_test.c +++ b/tests/thread_test.c @@ -70,6 +70,40 @@ static int s_test_thread_creation_join_fn(struct aws_allocator *allocator, void AWS_TEST_CASE(thread_creation_join_test, s_test_thread_creation_join_fn) +static int s_test_thread_creation_join_invalid_cpu_id_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + aws_common_library_init(allocator); + struct thread_test_data test_data = {.allocator = allocator}; + + struct aws_thread thread; + aws_thread_init(&thread, allocator); + + struct aws_thread_options thread_options = *aws_default_thread_options(); + /* invalid cpu_id. Ensure that the cpu_id is best-effort based. */ + thread_options.cpu_id = 4096; + + ASSERT_SUCCESS( + aws_thread_launch(&thread, s_thread_fn, (void *)&test_data, &thread_options), "thread creation failed"); + ASSERT_INT_EQUALS( + AWS_THREAD_JOINABLE, aws_thread_get_detach_state(&thread), "thread state should have returned JOINABLE"); + ASSERT_SUCCESS(aws_thread_join(&thread), "thread join failed"); + ASSERT_TRUE( + aws_thread_thread_id_equal(test_data.thread_id, aws_thread_get_id(&thread)), + "get_thread_id should have returned the same id as the thread calling current_thread_id"); + ASSERT_INT_EQUALS( + AWS_THREAD_JOIN_COMPLETED, + aws_thread_get_detach_state(&thread), + "thread state should have returned JOIN_COMPLETED"); + + aws_string_destroy(test_data.thread_name); + aws_thread_clean_up(&thread); + aws_common_library_clean_up(); + + return 0; +} + +AWS_TEST_CASE(thread_creation_join_invalid_cpu_id_test, s_test_thread_creation_join_invalid_cpu_id_fn) + static uint32_t s_atexit_call_count = 0; static void s_thread_atexit_fn(void *user_data) { (void)user_data; From 2fd6652adcfaa47eda1e0cbaa0c66cc9b72907fc Mon Sep 17 00:00:00 2001 From: Alfred G <28123637+alfred2g@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:35:26 -0700 Subject: [PATCH 07/42] Add support for WIN ARM64 (#1092) --- include/aws/common/atomics_msvc.inl | 80 +++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 16 deletions(-) diff --git a/include/aws/common/atomics_msvc.inl b/include/aws/common/atomics_msvc.inl index 516385f27..d1f3ba54c 100644 --- a/include/aws/common/atomics_msvc.inl +++ b/include/aws/common/atomics_msvc.inl @@ -20,8 +20,8 @@ AWS_EXTERN_C_BEGIN -#if !(defined(_M_IX86) || defined(_M_X64)) -# error Atomics are not currently supported for non-x86 MSVC platforms +#if !(defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64)) +# error Atomics are not currently supported for non-x86 or ARM64 MSVC platforms /* * In particular, it's not clear that seq_cst will work properly on non-x86 @@ -63,6 +63,24 @@ AWS_EXTERN_C_BEGIN * this use case. */ +/** + * Some general notes about ARM environments: + * ARM processors uses a weak memory model as opposed to the strong memory model used by Intel processors + * This means more permissible memory ordering allowed between stores and loads. + * + * Thus ARM port will need more hardware fences/barriers to assure developer intent. + * Memory barriers will prevent reordering stores and loads accross them depending on their type + * (read write, write only, read only ...) + * + * For more information about ARM64 memory ordering, + * see https://developer.arm.com/documentation/102336/0100/Memory-ordering + * For more information about Memory barriers, + * see https://developer.arm.com/documentation/102336/0100/Memory-barriers + * For more information about Miscosoft Interensic ARM64 APIs, + * see https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-170 + * Note: wrt _Interlocked[Op]64 is the same for ARM64 and x64 processors + */ + #ifdef _M_IX86 # define AWS_INTERLOCKED_INT(x) _Interlocked##x typedef long aws_atomic_impl_int_t; @@ -71,6 +89,32 @@ typedef long aws_atomic_impl_int_t; typedef long long aws_atomic_impl_int_t; #endif +#ifdef _M_ARM64 +/* Hardware Read Write barrier, prevents all memory operations to cross the barrier in both directions */ +# define AWS_RW_BARRIER() __dmb(_ARM64_BARRIER_SY) +/* Hardware Read barrier, prevents all memory operations to cross the barrier upwards */ +# define AWS_R_BARRIER() __dmb(_ARM64_BARRIER_LD) +/* Hardware Write barrier, prevents all memory operations to cross the barrier downwards */ +# define AWS_W_BARRIER() __dmb(_ARM64_BARRIER_ST) +/* Software barrier, prevents the compiler from reodering the operations across the barrier */ +# define AWS_SW_BARRIER() _ReadWriteBarrier(); +#else +/* hardware barriers, do nothing on x86 since it has a strong memory model + * as described in the section above: some general notes + */ +# define AWS_RW_BARRIER() +# define AWS_R_BARRIER() +# define AWS_W_BARRIER() +/* + * x86: only a compiler barrier is required. For seq_cst, we must use some form of interlocked operation for + * writes, but that's the caller's responsibility. + * + * Volatile ops may or may not imply this barrier, depending on the /volatile: switch, but adding an extra + * barrier doesn't hurt. + */ +# define AWS_SW_BARRIER() _ReadWriteBarrier(); /* software barrier */ +#endif + static inline void aws_atomic_priv_check_order(enum aws_memory_order order) { #ifndef NDEBUG switch (order) { @@ -107,14 +151,8 @@ static inline void aws_atomic_priv_barrier_before(enum aws_memory_order order, e return; } - /* - * x86: only a compiler barrier is required. For seq_cst, we must use some form of interlocked operation for - * writes, but that's the caller's responsibility. - * - * Volatile ops may or may not imply this barrier, depending on the /volatile: switch, but adding an extra - * barrier doesn't hurt. - */ - _ReadWriteBarrier(); + AWS_RW_BARRIER(); + AWS_SW_BARRIER(); } static inline void aws_atomic_priv_barrier_after(enum aws_memory_order order, enum aws_atomic_mode_priv mode) { @@ -131,11 +169,8 @@ static inline void aws_atomic_priv_barrier_after(enum aws_memory_order order, en return; } - /* - * x86: only a compiler barrier is required. For seq_cst, we must use some form of interlocked operation for - * writes, but that's the caller's responsibility. - */ - _ReadWriteBarrier(); + AWS_RW_BARRIER(); + AWS_SW_BARRIER(); } /** @@ -344,9 +379,16 @@ void aws_atomic_thread_fence(enum aws_memory_order order) { AWS_INTERLOCKED_INT(Exchange)(&x, 1); break; case aws_memory_order_release: + AWS_W_BARRIER(); + AWS_SW_BARRIER(); + break; case aws_memory_order_acquire: + AWS_R_BARRIER(); + AWS_SW_BARRIER(); + break; case aws_memory_order_acq_rel: - _ReadWriteBarrier(); + AWS_RW_BARRIER(); + AWS_SW_BARRIER(); break; case aws_memory_order_relaxed: /* no-op */ @@ -354,6 +396,12 @@ void aws_atomic_thread_fence(enum aws_memory_order order) { } } +/* prevent conflicts with other files that might pick the same names */ +#undef AWS_RW_BARRIER +#undef AWS_R_BARRIER +#undef AWS_W_BARRIER +#undef AWS_SW_BARRIER + #define AWS_ATOMICS_HAVE_THREAD_FENCE AWS_EXTERN_C_END #endif From 33c1bfbecd94dd79be2cd7f167d30e3090204ffd Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 21 Mar 2024 21:17:05 -0700 Subject: [PATCH 08/42] Better vectorization and crc64. Cleaned up cmake and added better runtime cpu detection (#1083) Co-authored-by: Alfred G <28123637+alfred2g@users.noreply.github.com> Co-authored-by: Alfred Gedeon --- CMakeLists.txt | 14 ++-- bin/system_info/print_system_info.c | 14 ++++ cmake/AwsFeatureTests.cmake | 13 +++ cmake/AwsSIMD.cmake | 97 +++++++++++++++++------ include/aws/common/config.h.in | 7 ++ include/aws/common/cpuid.h | 2 + source/arch/arm/{asm => auxv}/cpuid.c | 8 +- source/arch/arm/darwin/cpuid.c | 40 ++++++++++ source/arch/arm/{msvc => windows}/cpuid.c | 13 ++- source/arch/intel/cpuid.c | 4 +- 10 files changed, 178 insertions(+), 34 deletions(-) rename source/arch/arm/{asm => auxv}/cpuid.c (86%) create mode 100644 source/arch/arm/darwin/cpuid.c rename source/arch/arm/{msvc => windows}/cpuid.c (57%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e5c5cd9f..6d6e615a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,13 +152,17 @@ if (USE_CPU_EXTENSIONS) ) endif() elseif (AWS_ARCH_ARM64 OR AWS_ARCH_ARM32) - if (MSVC) + if (WINDOWS) file(GLOB AWS_COMMON_ARCH_SRC - "source/arch/arm/msvc/*.c" + "source/arch/arm/windows/*.c" ) - elseif (AWS_HAVE_AUXV) + elseif(APPLE) + file(GLOB AWS_COMMON_ARCH_SRC + "source/arch/arm/darwin/*.c" + ) + else() file(GLOB AWS_COMMON_ARCH_SRC - "source/arch/arm/asm/*.c" + "source/arch/arm/auxv/*.c" ) endif() endif() @@ -221,7 +225,7 @@ target_compile_definitions(${PROJECT_NAME} PRIVATE -DCJSON_HIDE_SYMBOLS) if (AWS_HAVE_AVX2_INTRINSICS) target_compile_definitions(${PROJECT_NAME} PRIVATE -DUSE_SIMD_ENCODING) - simd_add_source_avx(${PROJECT_NAME} "source/arch/intel/encoding_avx2.c") + simd_append_source_and_features(${PROJECT_NAME} "source/arch/intel/encoding_avx2.c" ${AWS_AVX2_FLAG}) message(STATUS "Building SIMD base64 decoder") endif() diff --git a/bin/system_info/print_system_info.c b/bin/system_info/print_system_info.c index c29877086..aac1b570f 100644 --- a/bin/system_info/print_system_info.c +++ b/bin/system_info/print_system_info.c @@ -3,6 +3,7 @@ #include #include #include +#include int main(void) { struct aws_allocator *allocator = aws_default_allocator(); @@ -39,6 +40,19 @@ int main(void) { fprintf(stdout, " 'numa architecture': 'false'\n"); } + fprintf(stdout, " 'cpu_capabilities': {\n"); + fprintf(stdout, " 'arm_crc': %s,\n", aws_cpu_has_feature(AWS_CPU_FEATURE_ARM_CRC) ? "true" : "false"); + fprintf(stdout, " 'arm_pmull': %s,\n", aws_cpu_has_feature(AWS_CPU_FEATURE_ARM_PMULL) ? "true" : "false"); + fprintf(stdout, " 'arm_crypto': %s,\n", aws_cpu_has_feature(AWS_CPU_FEATURE_ARM_CRYPTO) ? "true" : "false"); + fprintf(stdout, " 'amd_sse4_1': %s,\n", aws_cpu_has_feature(AWS_CPU_FEATURE_SSE_4_1) ? "true" : "false"); + fprintf(stdout, " 'amd_sse4_2': %s,\n", aws_cpu_has_feature(AWS_CPU_FEATURE_SSE_4_2) ? "true" : "false"); + fprintf(stdout, " 'amd_clmul': %s,\n", aws_cpu_has_feature(AWS_CPU_FEATURE_CLMUL) ? "true" : "false"); + fprintf(stdout, " 'amd_vpclmulqdq': %s,\n", aws_cpu_has_feature(AWS_CPU_FEATURE_VPCLMULQDQ) ? "true" : "false"); + fprintf(stdout, " 'amd_avx2': %s,\n", aws_cpu_has_feature(AWS_CPU_FEATURE_AVX2) ? "true" : "false"); + fprintf(stdout, " 'amd_avx512': %s,\n", aws_cpu_has_feature(AWS_CPU_FEATURE_AVX512) ? "true" : "false"); + fprintf(stdout, " 'amd_bmi2': %s\n", aws_cpu_has_feature(AWS_CPU_FEATURE_BMI2) ? "true" : "false"); + fprintf(stdout, " }\n"); + fprintf(stdout, "}\n"); aws_system_environment_release(env); aws_logger_clean_up(&logger); diff --git a/cmake/AwsFeatureTests.cmake b/cmake/AwsFeatureTests.cmake index 60a548cfd..813a3c9f8 100644 --- a/cmake/AwsFeatureTests.cmake +++ b/cmake/AwsFeatureTests.cmake @@ -17,6 +17,10 @@ if(MINGW) set(USE_CPU_EXTENSIONS OFF) endif() +if (USE_CPU_EXTENSIONS) + set(AWS_USE_CPU_EXTENSIONS ON) +endif() + if(NOT CMAKE_CROSSCOMPILING) check_c_source_runs(" #include @@ -54,6 +58,15 @@ check_c_source_compiles(" } " AWS_ARCH_INTEL) +check_c_source_compiles(" + int main() { +#if !(defined(__x86_64__) || defined(_M_X64)) +# error \"not intel\" +#endif + return 0; + } +" AWS_ARCH_INTEL_X64) + check_c_source_compiles(" int main() { #if !(defined(__aarch64__) || defined(_M_ARM64)) diff --git a/cmake/AwsSIMD.cmake b/cmake/AwsSIMD.cmake index 6ba9f236e..65fce96c7 100644 --- a/cmake/AwsSIMD.cmake +++ b/cmake/AwsSIMD.cmake @@ -4,35 +4,39 @@ include(CheckCCompilerFlag) include(CheckIncludeFile) +if (MSVC) + set(AWS_AVX2_FLAG "/arch:AVX2") + set(AWS_AVX512_FLAG "/arch:AVX512") + set(AWS_AVX512vL_FLAG "") + set(AWS_CLMUL_FLAG "") + set(AWS_SSE4_2_FLAG "") + set(AWS_ARMv8_1_FLAG "/arch:arm8.1") + set(WERROR_FLAG "") +else() + set(AWS_AVX2_FLAG "-mavx -mavx2") + set(AWS_AVX512_FLAG "-mavx512f -mvpclmulqdq") + set(AWS_AVX512vL_FLAG "-mavx512vl") + set(AWS_CLMUL_FLAG "-mpclmul") + set(AWS_SSE4_2_FLAG "-msse4.2") + set(AWS_ARMv8_1_FLAG "-march=armv8-a+crc+crypto -mtune=neoverse-v1") + set(WERROR_FLAG "-Werror") +endif() + if (USE_CPU_EXTENSIONS) - if (MSVC) - check_c_compiler_flag("/arch:AVX2" HAVE_M_AVX2_FLAG) - if (HAVE_M_AVX2_FLAG) - set(AVX_CFLAGS "/arch:AVX2") - endif() - else() - check_c_compiler_flag(-mavx2 HAVE_M_AVX2_FLAG) - if (HAVE_M_AVX2_FLAG) - set(AVX_CFLAGS "-mavx -mavx2") - endif() + set(AVX_CFLAGS ${AWS_SSE4_2_FLAG}) + + check_c_compiler_flag(${AWS_AVX2_FLAG} HAVE_M_AVX2_FLAG) + if (HAVE_M_AVX2_FLAG) + set(AVX_CFLAGS "${AWS_AVX2_FLAG} ${AVX_CFLAGS}") endif() - if (MSVC) - check_c_compiler_flag("/arch:AVX512" HAVE_M_AVX512_FLAG) - if (HAVE_M_AVX512_FLAG) - # docs imply AVX512 brings in AVX2. And it will compile, but it will break at runtime on - # instructions such as _mm256_load_si256(). Leave it on. - set(AVX_CFLAGS "/arch:AVX512 /arch:AVX2") - endif() - else() - check_c_compiler_flag("-mavx512f -mvpclmulqdq" HAVE_M_AVX512_FLAG) - if (HAVE_M_AVX512_FLAG) - set(AVX_CFLAGS "-mavx512f -mvpclmulqdq -mpclmul -mavx -mavx2 -msse4.2") - endif() + check_c_compiler_flag("${AWS_AVX512_FLAG} ${AWS_CLMUL_FLAG}" HAVE_M_AVX512_FLAG) + if (HAVE_M_AVX512_FLAG) + set(AVX_CFLAGS "${AWS_AVX512_FLAG} ${AWS_CLMUL_FLAG} ${AVX_CFLAGS}") endif() set(old_flags "${CMAKE_REQUIRED_FLAGS}") - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${AVX_CFLAGS}") + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${AVX_CFLAGS} ${WERROR_FLAG}") check_c_source_compiles(" #include @@ -68,7 +72,35 @@ if (USE_CPU_EXTENSIONS) return (int)_mm256_extract_epi64(vec, 2); }" AWS_HAVE_MM256_EXTRACT_EPI64) + check_c_source_compiles(" + #include + #include + int main() { + __m128i a = _mm_setzero_si128(); + __m128i b = _mm_setzero_si128(); + __m128i result = _mm_clmulepi64_si128(a, b, 0x00); + (void)result; + return 0; + }" AWS_HAVE_CLMUL) + + set(CMAKE_REQUIRED_FLAGS "${old_flags} ${AWS_ARMv8_1_FLAG} ${WERROR_FLAG}") + check_c_source_compiles(" + #include + int main() { + int crc = __crc32d(0, 1); + return 0; + }" AWS_HAVE_ARM32_CRC) + + check_c_source_compiles(" + #include + int main() { + _Atomic int var = 0; + atomic_fetch_add_explicit(&var, 1, memory_order_relaxed); + return 0; + }" AWS_HAVE_ARMv8_1) + set(CMAKE_REQUIRED_FLAGS "${old_flags}") + endif() # USE_CPU_EXTENSIONS # The part where the definition is added to the compiler flags has been moved to config.h.in @@ -80,6 +112,23 @@ endif() # USE_CPU_EXTENSIONS function(simd_add_source_avx target) foreach(file ${ARGN}) target_sources(${target} PRIVATE ${file}) - set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "${AVX_CFLAGS}") + set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS " ${AVX_CFLAGS}") endforeach() endfunction(simd_add_source_avx) + +# The part where the definition is added to the compiler flags has been moved to config.h.in +# see git history for more details. + +# Adds compiler flags to the source and adds the source to target. +# Unfortunately the flags have to be passed as strings. Predefined flags are +# at the top of this file. +# Usage: simd_append_source_and_features(target file1.c ${AWS_AVX512_FLAG} ${AWS_AVX2_FLAG} ...) +function(simd_append_source_and_features target file) + set(CC_FLAGS "") + foreach(flag ${ARGN}) + set(CC_FLAGS "${CC_FLAGS} ${flag}") + endforeach() + + target_sources(${target} PRIVATE ${file}) + set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS " ${CC_FLAGS}") +endfunction(simd_append_source_and_features) diff --git a/include/aws/common/config.h.in b/include/aws/common/config.h.in index d3dff3af2..381d99c99 100644 --- a/include/aws/common/config.h.in +++ b/include/aws/common/config.h.in @@ -22,5 +22,12 @@ #cmakedefine AWS_HAVE_AVX2_INTRINSICS #cmakedefine AWS_HAVE_AVX512_INTRINSICS #cmakedefine AWS_HAVE_MM256_EXTRACT_EPI64 +#cmakedefine AWS_HAVE_CLMUL +#cmakedefine AWS_HAVE_ARM32_CRC +#cmakedefine AWS_HAVE_ARMv8_1 +#cmakedefine AWS_ARCH_ARM64 +#cmakedefine AWS_ARCH_INTEL +#cmakedefine AWS_ARCH_INTEL_X64 +#cmakedefine AWS_USE_CPU_EXTENSIONS #endif diff --git a/include/aws/common/cpuid.h b/include/aws/common/cpuid.h index 84024a36b..9ab7d5059 100644 --- a/include/aws/common/cpuid.h +++ b/include/aws/common/cpuid.h @@ -18,6 +18,8 @@ enum aws_cpu_feature_name { AWS_CPU_FEATURE_ARM_CRC, AWS_CPU_FEATURE_BMI2, AWS_CPU_FEATURE_VPCLMULQDQ, + AWS_CPU_FEATURE_ARM_PMULL, + AWS_CPU_FEATURE_ARM_CRYPTO, AWS_CPU_FEATURE_COUNT, }; diff --git a/source/arch/arm/asm/cpuid.c b/source/arch/arm/auxv/cpuid.c similarity index 86% rename from source/arch/arm/asm/cpuid.c rename to source/arch/arm/auxv/cpuid.c index 6a306df98..10499da73 100644 --- a/source/arch/arm/asm/cpuid.c +++ b/source/arch/arm/auxv/cpuid.c @@ -29,7 +29,9 @@ struct cap_bits { # if (defined(__aarch64__)) struct cap_bits s_check_cap[AWS_CPU_FEATURE_COUNT] = { - [AWS_CPU_FEATURE_ARM_CRC] = {0, 1 << 7 /* HWCAP_CRC */}, + [AWS_CPU_FEATURE_ARM_CRC] = {0, 1 << 7 /* HWCAP_CRC32 */}, + [AWS_CPU_FEATURE_ARM_PMULL] = {0, 1 << 4 /* HWCAP_PMULL */}, + [AWS_CPU_FEATURE_ARM_CRYPTO] = {0, 1 << 3 /* HWCAP_AES */}, }; # else struct cap_bits s_check_cap[AWS_CPU_FEATURE_COUNT] = { @@ -67,6 +69,10 @@ bool aws_cpu_has_feature(enum aws_cpu_feature_name feature_name) { switch (feature_name) { case AWS_CPU_FEATURE_ARM_CRC: +# if (defined(__aarch64__)) + case AWS_CPU_FEATURE_ARM_PMULL: + case AWS_CPU_FEATURE_ARM_CRYPTO: +# endif // (defined(__aarch64__)) return s_hwcap[s_check_cap[feature_name].cap] & s_check_cap[feature_name].bit; default: return false; diff --git a/source/arch/arm/darwin/cpuid.c b/source/arch/arm/darwin/cpuid.c new file mode 100644 index 000000000..7552d4220 --- /dev/null +++ b/source/arch/arm/darwin/cpuid.c @@ -0,0 +1,40 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +#include + +#include + +bool aws_cpu_has_feature(enum aws_cpu_feature_name feature_name) { + int64_t ret = 0; + size_t size = sizeof(ret); + + switch (feature_name) { + case AWS_CPU_FEATURE_ARM_PMULL: + if (sysctlbyname("hw.optional.arm.FEAT_PMULL", &ret, &size, NULL, 0) != -1) { + return ret == 1; + } + case AWS_CPU_FEATURE_ARM_CRC: + if (sysctlbyname("hw.optional.armv8_crc32", &ret, &size, NULL, 0) != -1) { + return ret == 1; + } + case AWS_CPU_FEATURE_ARM_CRYPTO: + if (sysctlbyname("hw.optional.arm.FEAT_AES", &ret, &size, NULL, 0) != -1) { + return ret == 1; + } + default: + return false; + } +} diff --git a/source/arch/arm/msvc/cpuid.c b/source/arch/arm/windows/cpuid.c similarity index 57% rename from source/arch/arm/msvc/cpuid.c rename to source/arch/arm/windows/cpuid.c index c10c5d15c..b7da0053f 100644 --- a/source/arch/arm/msvc/cpuid.c +++ b/source/arch/arm/windows/cpuid.c @@ -13,9 +13,18 @@ * permissions and limitations under the License. */ +#include #include -#include bool aws_cpu_has_feature(enum aws_cpu_feature_name feature_name) { - return false; + switch (feature_name) { + case AWS_CPU_FEATURE_ARM_CRC: + return IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) != 0; + // this is the best we've got on windows as they don't separate PMULL and AES from each other. + case AWS_CPU_FEATURE_ARM_PMULL: + case AWS_CPU_FEATURE_ARM_CRYPTO: + return IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != 0; + default: + return false; + } } diff --git a/source/arch/intel/cpuid.c b/source/arch/intel/cpuid.c index 44fdff078..465fccd17 100644 --- a/source/arch/intel/cpuid.c +++ b/source/arch/intel/cpuid.c @@ -116,8 +116,8 @@ static bool s_has_bmi2(void) { static bool s_has_vpclmulqdq(void) { uint32_t abcd[4]; /* Check VPCLMULQDQ: - * CPUID.(EAX=07H, ECX=0H):ECX.VPCLMULQDQ[bit 20]==1 */ - uint32_t vpclmulqdq_mask = (1 << 20); + * CPUID.(EAX=07H, ECX=0H):ECX.VPCLMULQDQ[bit 10]==1 */ + uint32_t vpclmulqdq_mask = (1 << 10); aws_run_cpuid(7, 0, abcd); if ((abcd[2] & vpclmulqdq_mask) != vpclmulqdq_mask) { return false; From ae7b067d9274d2d3faa1d3ae42d489a6986661f7 Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:36:01 -0700 Subject: [PATCH 09/42] Avoid overallocating for strings (#1099) --- source/string.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/source/string.c b/source/string.c index a3d2c204e..2fd791230 100644 --- a/source/string.c +++ b/source/string.c @@ -189,11 +189,8 @@ struct aws_string *aws_string_new_from_c_str(struct aws_allocator *allocator, co struct aws_string *aws_string_new_from_array(struct aws_allocator *allocator, const uint8_t *bytes, size_t len) { AWS_PRECONDITION(allocator); AWS_PRECONDITION(AWS_MEM_IS_READABLE(bytes, len)); - size_t malloc_size; - if (aws_add_size_checked(sizeof(struct aws_string) + 1, len, &malloc_size)) { - return NULL; - } - struct aws_string *str = aws_mem_acquire(allocator, malloc_size); + + struct aws_string *str = aws_mem_acquire(allocator, offsetof(struct aws_string, bytes[len + 1])); if (!str) { return NULL; } From 656a2ef19f94eab99981673f85760e6a072c50df Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Fri, 12 Apr 2024 14:47:12 -0700 Subject: [PATCH 10/42] Switch test harness to print errors in decimal (#1100) --- include/aws/testing/aws_test_harness.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/aws/testing/aws_test_harness.h b/include/aws/testing/aws_test_harness.h index d4d258b2c..3c18e1ab0 100644 --- a/include/aws/testing/aws_test_harness.h +++ b/include/aws/testing/aws_test_harness.h @@ -147,7 +147,7 @@ static int s_cunit_failure_message0( if (assert_rv != AWS_OP_SUCCESS) { \ if (!PRINT_FAIL_INTERNAL0(__VA_ARGS__)) { \ PRINT_FAIL_INTERNAL0( \ - "Expected success at %s; got return value %d with last error 0x%04x\n", \ + "Expected success at %s; got return value %d with last error 0x%04d\n", \ #condition, \ assert_rv, \ aws_last_error()); \ @@ -162,7 +162,7 @@ static int s_cunit_failure_message0( if (assert_rv != AWS_OP_ERR) { \ if (!PRINT_FAIL_INTERNAL0(__VA_ARGS__)) { \ PRINT_FAIL_INTERNAL0( \ - "Expected failure at %s; got return value %d with last error 0x%04x\n", \ + "Expected failure at %s; got return value %d with last error 0x%04d\n", \ #condition, \ assert_rv, \ aws_last_error()); \ @@ -179,7 +179,7 @@ static int s_cunit_failure_message0( if (assert_rv != AWS_OP_ERR) { \ fprintf( \ AWS_TESTING_REPORT_FD, \ - "%sExpected error but no error occurred; rv=%d, aws_last_error=%04x (expected %04x): ", \ + "%sExpected error but no error occurred; rv=%d, aws_last_error=%04d (expected %04d): ", \ FAIL_PREFIX, \ assert_rv, \ assert_err, \ @@ -192,7 +192,7 @@ static int s_cunit_failure_message0( if (assert_err != assert_err_expect) { \ fprintf( \ AWS_TESTING_REPORT_FD, \ - "%sIncorrect error code; aws_last_error=%04x (expected %04x): ", \ + "%sIncorrect error code; aws_last_error=%04d (expected %04d): ", \ FAIL_PREFIX, \ assert_err, \ assert_err_expect); \ From ad15e6a29208fdd856f57b429196a249d00fbad7 Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Mon, 15 Apr 2024 12:53:49 -0700 Subject: [PATCH 11/42] Add list membership API and uint64 hash utilities (#1103) --- include/aws/common/hash_table.h | 10 ++++++++++ include/aws/common/linked_list.h | 5 +++++ include/aws/common/linked_list.inl | 4 ++++ source/hash_table.c | 8 ++++++++ tests/CMakeLists.txt | 1 + tests/linked_list_test.c | 23 +++++++++++++++++++++++ 6 files changed, 51 insertions(+) diff --git a/include/aws/common/hash_table.h b/include/aws/common/hash_table.h index a23c4f966..d28868f21 100644 --- a/include/aws/common/hash_table.h +++ b/include/aws/common/hash_table.h @@ -434,6 +434,16 @@ bool aws_hash_table_is_valid(const struct aws_hash_table *map); AWS_COMMON_API bool aws_hash_iter_is_valid(const struct aws_hash_iter *iter); +/** + * Helper function to hash keys that are uint64_t values. + */ +AWS_COMMON_API uint64_t aws_hash_uint64_t(const void *item); + +/** + * Helper function to compare hash keys that are uint64_t values. + */ +AWS_COMMON_API bool aws_hash_compare_uint64_t_eq(const void *a, const void *b); + AWS_EXTERN_C_END AWS_POP_SANE_WARNING_LEVEL diff --git a/include/aws/common/linked_list.h b/include/aws/common/linked_list.h index 3f550c6fe..cb0ce655c 100644 --- a/include/aws/common/linked_list.h +++ b/include/aws/common/linked_list.h @@ -182,6 +182,11 @@ AWS_STATIC_IMPL void aws_linked_list_move_all_front( struct aws_linked_list *AWS_RESTRICT dst, struct aws_linked_list *AWS_RESTRICT src); +/** + * Returns true if the node is currently in a list, false otherwise. + */ +AWS_STATIC_IMPL bool aws_linked_list_node_is_in_list(struct aws_linked_list_node *node); + #ifndef AWS_NO_STATIC_IMPL # include #endif /* AWS_NO_STATIC_IMPL */ diff --git a/include/aws/common/linked_list.inl b/include/aws/common/linked_list.inl index 99604834a..4d7e54dd0 100644 --- a/include/aws/common/linked_list.inl +++ b/include/aws/common/linked_list.inl @@ -431,6 +431,10 @@ AWS_STATIC_IMPL void aws_linked_list_move_all_front( AWS_POSTCONDITION(aws_linked_list_is_valid(dst)); } +AWS_STATIC_IMPL bool aws_linked_list_node_is_in_list(struct aws_linked_list_node *node) { + return aws_linked_list_node_prev_is_valid(node) && aws_linked_list_node_next_is_valid(node); +} + AWS_EXTERN_C_END #endif /* AWS_COMMON_LINKED_LIST_INL */ diff --git a/source/hash_table.c b/source/hash_table.c index 21f989a84..bf559c7b8 100644 --- a/source/hash_table.c +++ b/source/hash_table.c @@ -1104,3 +1104,11 @@ int hash_table_state_required_bytes(size_t size, size_t *required_bytes) { return AWS_OP_SUCCESS; } + +uint64_t aws_hash_uint64_t(const void *item) { + return *(uint64_t *)item; +} + +bool aws_hash_compare_uint64_t_eq(const void *a, const void *b) { + return *(uint64_t *)a == *(uint64_t *)b; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d2462469a..35be1133d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -105,6 +105,7 @@ add_test_case(linked_list_reverse_iteration) add_test_case(linked_list_swap_contents) add_test_case(linked_list_move_all_back) add_test_case(linked_list_move_all_front) +add_test_case(linked_list_node_in_list) add_test_case(hex_encoding_test_case_empty_test) add_test_case(hex_encoding_test_case_f_test) diff --git a/tests/linked_list_test.c b/tests/linked_list_test.c index d9226c602..2b29a5645 100644 --- a/tests/linked_list_test.c +++ b/tests/linked_list_test.c @@ -518,3 +518,26 @@ AWS_TEST_CASE(linked_list_reverse_iteration, s_test_linked_list_reverse_iteratio AWS_TEST_CASE(linked_list_swap_contents, s_test_linked_list_swap_contents) AWS_TEST_CASE(linked_list_move_all_back, s_test_linked_list_move_all_back) AWS_TEST_CASE(linked_list_move_all_front, s_test_linked_list_move_all_front) + +static int s_linked_list_node_in_list_fn(struct aws_allocator *allocator, void *ctx) { + (void)allocator; + (void)ctx; + + struct aws_linked_list list; + aws_linked_list_init(&list); + + struct aws_linked_list_node node; + AWS_ZERO_STRUCT(node); + + ASSERT_FALSE(aws_linked_list_node_is_in_list(&node)); + + aws_linked_list_push_back(&list, &node); + ASSERT_TRUE(aws_linked_list_node_is_in_list(&node)); + aws_linked_list_remove(&node); + + ASSERT_FALSE(aws_linked_list_node_is_in_list(&node)); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(linked_list_node_in_list, s_linked_list_node_in_list_fn) From 86e9e27a23148e9468c15cae9cd23a7cbc74b542 Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Tue, 16 Apr 2024 10:44:26 -0700 Subject: [PATCH 12/42] Clarity updates for uint64_t hash helper (#1104) Co-authored-by: Bret Ambrose --- include/aws/common/hash_table.h | 6 +++++- source/hash_table.c | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/aws/common/hash_table.h b/include/aws/common/hash_table.h index d28868f21..c2cfadd77 100644 --- a/include/aws/common/hash_table.h +++ b/include/aws/common/hash_table.h @@ -436,8 +436,12 @@ bool aws_hash_iter_is_valid(const struct aws_hash_iter *iter); /** * Helper function to hash keys that are uint64_t values. + * + * The function is not a strong hash function in any sense; it merely reflects + * the uint64 value back. Do not use this function as a hash if you need + * the properties of a strong hash function. */ -AWS_COMMON_API uint64_t aws_hash_uint64_t(const void *item); +AWS_COMMON_API uint64_t aws_hash_uint64_t_by_identity(const void *item); /** * Helper function to compare hash keys that are uint64_t values. diff --git a/source/hash_table.c b/source/hash_table.c index bf559c7b8..bd5d04a3f 100644 --- a/source/hash_table.c +++ b/source/hash_table.c @@ -1105,7 +1105,7 @@ int hash_table_state_required_bytes(size_t size, size_t *required_bytes) { return AWS_OP_SUCCESS; } -uint64_t aws_hash_uint64_t(const void *item) { +uint64_t aws_hash_uint64_t_by_identity(const void *item) { return *(uint64_t *)item; } From 70b8fcd3c7a931370a6686559ba3e5996434136c Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 22 Apr 2024 09:31:41 -0700 Subject: [PATCH 13/42] Move is ipv4/ipv6 functions from Aws-c-sdkutils to Aws-c-common (#1105) --- include/aws/common/host_utils.h | 28 +++++++ source/host_utils.c | 127 ++++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 2 + tests/host_util_test.c | 74 +++++++++++++++++++ 4 files changed, 231 insertions(+) create mode 100644 include/aws/common/host_utils.h create mode 100644 source/host_utils.c create mode 100644 tests/host_util_test.c diff --git a/include/aws/common/host_utils.h b/include/aws/common/host_utils.h new file mode 100644 index 000000000..0b8285d7a --- /dev/null +++ b/include/aws/common/host_utils.h @@ -0,0 +1,28 @@ +#ifndef AWS_COMMON_HOST_UTILS_H +#define AWS_COMMON_HOST_UTILS_H +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include + +struct aws_byte_cursor; + +AWS_PUSH_SANE_WARNING_LEVEL +AWS_EXTERN_C_BEGIN + +/* + * Determine whether host cursor is IPv4 string. + */ +AWS_COMMON_API bool aws_host_utils_is_ipv4(struct aws_byte_cursor host); + +/* + * Determine whether host cursor is IPv6 string. + * Supports checking for uri encoded strings and scoped literals. + */ +AWS_COMMON_API bool aws_host_utils_is_ipv6(struct aws_byte_cursor host, bool is_uri_encoded); + +AWS_EXTERN_C_END +AWS_POP_SANE_WARNING_LEVEL + +#endif /* AWS_COMMON_HOST_UTILS_H */ diff --git a/source/host_utils.c b/source/host_utils.c new file mode 100644 index 000000000..6cd26ba88 --- /dev/null +++ b/source/host_utils.c @@ -0,0 +1,127 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include +#include + +#ifdef _MSC_VER /* Disable sscanf warnings on windows. */ +# pragma warning(disable : 4204) +# pragma warning(disable : 4706) +# pragma warning(disable : 4996) +#endif + +/* 4 octets of 3 chars max + 3 separators + null terminator */ +#define AWS_IPV4_STR_LEN 16 +#define IP_CHAR_FMT "%03" SCNu16 + +static bool s_is_ipv6_char(uint8_t value) { + return aws_isxdigit(value) || value == ':'; +} + +static bool s_starts_with(struct aws_byte_cursor cur, uint8_t ch) { + return cur.len > 0 && cur.ptr[0] == ch; +} + +static bool s_ends_with(struct aws_byte_cursor cur, uint8_t ch) { + return cur.len > 0 && cur.ptr[cur.len - 1] == ch; +} + +bool aws_host_utils_is_ipv4(struct aws_byte_cursor host) { + if (host.len > AWS_IPV4_STR_LEN - 1) { + return false; + } + + char copy[AWS_IPV4_STR_LEN] = {0}; + memcpy(copy, host.ptr, host.len); + + uint16_t octet[4] = {0}; + char remainder[2] = {0}; + if (4 != sscanf( + copy, + IP_CHAR_FMT "." IP_CHAR_FMT "." IP_CHAR_FMT "." IP_CHAR_FMT "%1s", + &octet[0], + &octet[1], + &octet[2], + &octet[3], + remainder)) { + return false; + } + + for (size_t i = 0; i < 4; ++i) { + if (octet[i] > 255) { + return false; + } + } + + return true; +} + +/* actual encoding is %25, but % is omitted for simplicity, since split removes it */ +static struct aws_byte_cursor s_percent_uri_enc = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("25"); +/* + * IPv6 format: + * 8 groups of 4 hex chars separated by colons (:) + * leading 0s in each group can be skipped + * 2 or more consecutive zero groups can be replaced by double colon (::), + * but only once. + * ipv6 literal can be scoped by to zone by appending % followed by zone name + * ( does not look like there is length reqs on zone name length. this + * implementation enforces that its > 1 ) + * ipv6 can be embedded in url, in which case it must be wrapped inside [] + * and % be uri encoded as %25. + * Implementation is fairly trivial and just iterates through the string + * keeping track of the spec above. + */ +bool aws_host_utils_is_ipv6(struct aws_byte_cursor host, bool is_uri_encoded) { + if (host.len == 0) { + return false; + } + + if (is_uri_encoded) { + if (!s_starts_with(host, '[') || !s_ends_with(host, ']')) { + return false; + } + aws_byte_cursor_advance(&host, 1); + --host.len; + } + + struct aws_byte_cursor substr = {0}; + /* first split is required ipv6 part */ + bool is_split = aws_byte_cursor_next_split(&host, '%', &substr); + AWS_ASSERT(is_split); /* function is guaranteed to return at least one split */ + + if (!is_split || substr.len == 0 || s_ends_with(substr, ':') || + !aws_byte_cursor_satisfies_pred(&substr, s_is_ipv6_char)) { + return false; + } + + uint8_t group_count = 0; + bool has_double_colon = false; + struct aws_byte_cursor group = {0}; + while (aws_byte_cursor_next_split(&substr, ':', &group)) { + ++group_count; + + if (group_count > 8 || /* too many groups */ + group.len > 4 || /* too many chars in group */ + (has_double_colon && group.len == 0 && group_count > 2)) { /* only one double colon allowed */ + return false; + } + + has_double_colon = has_double_colon || group.len == 0; + } + + /* second split is optional zone part */ + if (aws_byte_cursor_next_split(&host, '%', &substr)) { + if ((is_uri_encoded && + (substr.len < 3 || + !aws_byte_cursor_starts_with(&substr, &s_percent_uri_enc))) || /* encoding for % + 1 extra char */ + (!is_uri_encoded && substr.len == 0) || /* at least 1 char */ + !aws_byte_cursor_satisfies_pred(&substr, aws_isalnum)) { + return false; + } + } + + return has_double_colon ? group_count < 7 : group_count == 8; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 35be1133d..6f224e4f2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -528,6 +528,8 @@ add_test_case(test_cross_process_lock_works_cross_proc) add_test_case(cross_process_lock_mp_test_runner) add_test_case(test_cross_process_lock_invalid_nonce_fails) +add_test_case(host_util_is_ipv4) +add_test_case(host_util_is_ipv6) generate_test_driver(${PROJECT_NAME}-tests) diff --git a/tests/host_util_test.c b/tests/host_util_test.c new file mode 100644 index 000000000..bf0192610 --- /dev/null +++ b/tests/host_util_test.c @@ -0,0 +1,74 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include +#include + +AWS_TEST_CASE(host_util_is_ipv4, s_test_is_ipv4) +static int s_test_is_ipv4(struct aws_allocator *allocator, void *ctx) { + (void)allocator; + (void)ctx; + + ASSERT_TRUE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("0.0.0.0"))); + ASSERT_TRUE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("127.0.0.1"))); + ASSERT_TRUE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("255.255.255.255"))); + ASSERT_TRUE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("192.168.1.1"))); + + ASSERT_FALSE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("256.0.0.1"))); + ASSERT_FALSE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("127.0.0"))); + ASSERT_FALSE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("127.0"))); + ASSERT_FALSE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("127"))); + ASSERT_FALSE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str(""))); + + ASSERT_FALSE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("foo.com"))); + ASSERT_FALSE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("a.b.c.d"))); + ASSERT_FALSE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("a127.0.0.1"))); + ASSERT_FALSE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("127.0.0.1a"))); + ASSERT_FALSE(aws_host_utils_is_ipv4(aws_byte_cursor_from_c_str("127.0.0.1011"))); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(host_util_is_ipv6, s_test_is_ipv6) +static int s_test_is_ipv6(struct aws_allocator *allocator, void *ctx) { + (void)allocator; + (void)ctx; + + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("0:0:0000:0000:0000:0:0:0"), false)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("2001:0db8:0000:0000:0000:8a2e:0370:7334"), false)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("2001:0DB8:0000:0000:0000:8a2e:0370:7334"), false)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1"), false)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1%en0"), false)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("::1"), false)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("0:0:0:0:0:0:0:1"), false)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fd00:ec2::23"), false)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fd00:ec2:0:0:0:0:0:23"), false)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[2001:0db8:0000:0000:0000:8a2e:0370:7334]"), true)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1]"), true)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1%25en0]"), true)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[2001:db8:85a3:8d3:1319:8a2e:370:7348]"), true)); + + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("2001:0db8:0000:0000:0000:8a2e:0370"), false)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("2001:0db8:0000:0000:0000:8a2e:0370:"), false)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("2001::"), false)); + ASSERT_FALSE( + aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("2001:0db8:0000:0000:0000:8a2e:0370:7334:8745"), false)); + ASSERT_FALSE( + aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str(":2001:0db8:0000:0000:0000:8a2e:0370:7334:8745"), false)); + ASSERT_FALSE( + aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("z001:0db8:0000:0000:0000:8a2e:0370:7334:8745"), false)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("z001::8a2e::8745"), false)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("::2001:0db8:0000:0000:8a2e:0370:7334"), false)); + + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1%25en0"), true)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1%en0]"), true)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1%24en0]"), true)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1%25en0"), true)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1%25en0]"), true)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1%25]"), true)); + + return AWS_OP_SUCCESS; +} From 5038bc131178ef0ad7e1ef0bc2a8dc4425943c59 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 22 Apr 2024 11:12:15 -0700 Subject: [PATCH 14/42] Fix invalid cpu_id for RHEL_5 (#1106) --- tests/thread_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/thread_test.c b/tests/thread_test.c index 50085d843..9d9fb62d9 100644 --- a/tests/thread_test.c +++ b/tests/thread_test.c @@ -80,7 +80,7 @@ static int s_test_thread_creation_join_invalid_cpu_id_fn(struct aws_allocator *a struct aws_thread_options thread_options = *aws_default_thread_options(); /* invalid cpu_id. Ensure that the cpu_id is best-effort based. */ - thread_options.cpu_id = 4096; + thread_options.cpu_id = 512; ASSERT_SUCCESS( aws_thread_launch(&thread, s_thread_fn, (void *)&test_data, &thread_options), "thread creation failed"); From 85c08731a6a34df1df100b06fcf827fcef65cf01 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Mon, 22 Apr 2024 18:06:53 -0600 Subject: [PATCH 15/42] memtrace: Fix underflow when stack_depth < FRAMES_TO_SKIP (#873) Co-authored-by: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> --- source/memtrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/memtrace.c b/source/memtrace.c index 9c5bff60c..fbdcf86df 100644 --- a/source/memtrace.c +++ b/source/memtrace.c @@ -134,7 +134,7 @@ static void s_alloc_tracer_track(struct alloc_tracer *tracer, void *ptr, size_t /* capture stack frames, skip 2 for this function and the allocation vtable function */ AWS_VARIABLE_LENGTH_ARRAY(void *, stack_frames, (FRAMES_TO_SKIP + tracer->frames_per_stack)); size_t stack_depth = aws_backtrace(stack_frames, FRAMES_TO_SKIP + tracer->frames_per_stack); - if (stack_depth) { + if (stack_depth >= FRAMES_TO_SKIP) { /* hash the stack pointers */ struct aws_byte_cursor stack_cursor = aws_byte_cursor_from_array(stack_frames, stack_depth * sizeof(void *)); From 24da21d2fc46c6e172496f17aade5683a32b76bb Mon Sep 17 00:00:00 2001 From: Michael Tautschnig Date: Tue, 23 Apr 2024 23:17:15 +0200 Subject: [PATCH 16/42] chore: make CBMC stubs with zero parameters proper declarations (#1107) --- .../proof_helpers/aws_byte_cursor_read_common.h | 2 +- .../proof_helpers/make_common_data_structures.h | 2 +- verification/cbmc/include/proof_helpers/nondet.h | 16 ++++++++-------- .../cbmc/sources/make_common_data_structures.c | 2 +- .../cbmc/stubs/abort_override_assert_false.c | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/verification/cbmc/include/proof_helpers/aws_byte_cursor_read_common.h b/verification/cbmc/include/proof_helpers/aws_byte_cursor_read_common.h index ab159dcbd..2ff5b8919 100644 --- a/verification/cbmc/include/proof_helpers/aws_byte_cursor_read_common.h +++ b/verification/cbmc/include/proof_helpers/aws_byte_cursor_read_common.h @@ -6,7 +6,7 @@ #include #include -void aws_byte_cursor_read_common_harness() { +void aws_byte_cursor_read_common_harness(void) { /* parameters */ struct aws_byte_cursor cur; DEST_TYPE *dest = malloc(sizeof(*dest)); diff --git a/verification/cbmc/include/proof_helpers/make_common_data_structures.h b/verification/cbmc/include/proof_helpers/make_common_data_structures.h index dde63ae35..139a9f7f5 100644 --- a/verification/cbmc/include/proof_helpers/make_common_data_structures.h +++ b/verification/cbmc/include/proof_helpers/make_common_data_structures.h @@ -127,7 +127,7 @@ void hash_proof_destroy_noop(void *p); /** * Ensures a valid string is allocated, with as much nondet as possible */ -struct aws_string *ensure_string_is_allocated_nondet_length(); +struct aws_string *ensure_string_is_allocated_nondet_length(void); /** * Ensures a valid string is allocated, with as much nondet as possible, but len < max diff --git a/verification/cbmc/include/proof_helpers/nondet.h b/verification/cbmc/include/proof_helpers/nondet.h index 2fc4ac306..5a960ba3e 100644 --- a/verification/cbmc/include/proof_helpers/nondet.h +++ b/verification/cbmc/include/proof_helpers/nondet.h @@ -12,11 +12,11 @@ /** * Non-determinstic functions used in CBMC proofs */ -bool nondet_bool(); -int nondet_int(); -size_t nondet_size_t(); -uint16_t nondet_uint16_t(); -uint32_t nondet_uint32_t(); -uint64_t nondet_uint64_t(); -uint8_t nondet_uint8_t(); -void *nondet_voidp(); +bool nondet_bool(void); +int nondet_int(void); +size_t nondet_size_t(void); +uint16_t nondet_uint16_t(void); +uint32_t nondet_uint32_t(void); +uint64_t nondet_uint64_t(void); +uint8_t nondet_uint8_t(void); +void *nondet_voidp(void); diff --git a/verification/cbmc/sources/make_common_data_structures.c b/verification/cbmc/sources/make_common_data_structures.c index b2e4ef457..16bfb467d 100644 --- a/verification/cbmc/sources/make_common_data_structures.c +++ b/verification/cbmc/sources/make_common_data_structures.c @@ -189,7 +189,7 @@ bool hash_table_state_has_an_empty_slot(const struct hash_table_state *const sta */ void hash_proof_destroy_noop(void *p) {} -struct aws_string *ensure_string_is_allocated_nondet_length() { +struct aws_string *ensure_string_is_allocated_nondet_length(void) { /* Considers any size up to the maximum possible size for the array [bytes] in aws_string */ return nondet_allocate_string_bounded_length(SIZE_MAX - 1 - sizeof(struct aws_string)); } diff --git a/verification/cbmc/stubs/abort_override_assert_false.c b/verification/cbmc/stubs/abort_override_assert_false.c index 79798c3bc..e3b3a4b8a 100644 --- a/verification/cbmc/stubs/abort_override_assert_false.c +++ b/verification/cbmc/stubs/abort_override_assert_false.c @@ -11,6 +11,6 @@ #include -void abort() { +void abort(void) { assert(0); } From 86a056c11a285512e89837bb4aa02d8d8b85b1d4 Mon Sep 17 00:00:00 2001 From: Gilberto Lopez Zayas Date: Mon, 29 Apr 2024 10:50:50 -0700 Subject: [PATCH 17/42] satisfy some signed vs unsigned comparison warnings (#809) Co-authored-by: Lopez Co-authored-by: Michael Graeb Co-authored-by: Bret Ambrose Co-authored-by: Joseph Klix --- source/arch/intel/encoding_avx2.c | 8 ++++---- source/encoding.c | 4 ++-- tests/atomics_test.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/arch/intel/encoding_avx2.c b/source/arch/intel/encoding_avx2.c index 439d6ddad..92c4abdfa 100644 --- a/source/arch/intel/encoding_avx2.c +++ b/source/arch/intel/encoding_avx2.c @@ -194,13 +194,13 @@ static inline bool decode(const unsigned char *in, unsigned char *out) { size_t aws_common_private_base64_decode_sse41(const unsigned char *in, unsigned char *out, size_t len) { if (len % 4) { - return (size_t)-1; + return SIZE_MAX; } size_t outlen = 0; while (len > 32) { if (!decode(in, out)) { - return (size_t)-1; + return SIZE_MAX; } len -= 32; in += 32; @@ -230,13 +230,13 @@ size_t aws_common_private_base64_decode_sse41(const unsigned char *in, unsigned } if (!decode(tmp_in, tmp_out)) { - return (size_t)-1; + return SIZE_MAX; } /* Check that there are no trailing ones bits */ for (size_t i = final_out; i < sizeof(tmp_out); i++) { if (tmp_out[i]) { - return (size_t)-1; + return SIZE_MAX; } } diff --git a/source/encoding.c b/source/encoding.c index bc860aa4b..97b99145d 100644 --- a/source/encoding.c +++ b/source/encoding.c @@ -23,7 +23,7 @@ static inline size_t aws_common_private_base64_decode_sse41(const unsigned char (void)out; (void)len; AWS_ASSERT(false); - return (size_t)-1; /* unreachable */ + return SIZE_MAX; /* unreachable */ } static inline void aws_common_private_base64_encode_sse41(const unsigned char *in, unsigned char *out, size_t len) { (void)in; @@ -361,7 +361,7 @@ int aws_base64_decode(const struct aws_byte_cursor *AWS_RESTRICT to_decode, stru if (aws_common_private_has_avx2()) { size_t result = aws_common_private_base64_decode_sse41(to_decode->ptr, output->buffer, to_decode->len); - if (result == -1) { + if (result == SIZE_MAX) { return aws_raise_error(AWS_ERROR_INVALID_BASE64_STR); } diff --git a/tests/atomics_test.c b/tests/atomics_test.c index f1cf15848..16590a794 100644 --- a/tests/atomics_test.c +++ b/tests/atomics_test.c @@ -272,7 +272,7 @@ static int run_races( int *participant_indexes = alloca(n_participants_local * sizeof(*participant_indexes)); struct aws_thread *threads = alloca(n_participants_local * sizeof(struct aws_thread)); - *last_race = (size_t)-1; + *last_race = SIZE_MAX; n_participants = n_participants_local; done_racing = false; aws_atomic_init_int(&last_race_index, 0); @@ -293,7 +293,7 @@ static int run_races( *last_race = n_races; } else { *last_race = (size_t)aws_atomic_load_int_explicit(&last_race_index, aws_memory_order_relaxed); - if (*last_race == (size_t)-1) { + if (*last_race == SIZE_MAX) { /* We didn't even see the first race complete */ *last_race = 0; } From 6236599339e5571b6dbc4b5f2831b1913aa2e69a Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Fri, 3 May 2024 14:23:59 -0700 Subject: [PATCH 18/42] Remove promise class (#1110) --- include/aws/common/promise.h | 103 -------------------- source/promise.c | 118 ----------------------- tests/CMakeLists.txt | 6 -- tests/promise_test.c | 181 ----------------------------------- 4 files changed, 408 deletions(-) delete mode 100644 include/aws/common/promise.h delete mode 100644 source/promise.c delete mode 100644 tests/promise_test.c diff --git a/include/aws/common/promise.h b/include/aws/common/promise.h deleted file mode 100644 index 37f930ee1..000000000 --- a/include/aws/common/promise.h +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -#ifndef AWS_COMMON_PROMISE_H -#define AWS_COMMON_PROMISE_H - -#include - -AWS_PUSH_SANE_WARNING_LEVEL - -/* - * Standard promise interface. Promise can be waited on by multiple threads, and as long as it is - * ref-counted correctly, will provide the resultant value/error code to all waiters. - * All promise API calls are internally thread-safe. - */ -struct aws_promise; - -AWS_EXTERN_C_BEGIN - -/* - * Creates a new promise - */ -AWS_COMMON_API -struct aws_promise *aws_promise_new(struct aws_allocator *allocator); - -/* - * Indicate a new reference to a promise. At minimum, each new thread making use of the promise should - * acquire it. - */ -AWS_COMMON_API -struct aws_promise *aws_promise_acquire(struct aws_promise *promise); - -/* - * Releases a reference on the promise. When the refcount hits 0, the promise is cleaned up and freed. - */ -AWS_COMMON_API -void aws_promise_release(struct aws_promise *promise); - -/* - * Waits infinitely for the promise to be completed - */ -AWS_COMMON_API -void aws_promise_wait(struct aws_promise *promise); -/* - * Waits for the requested time in nanoseconds. Returns true if the promise was completed. - */ -AWS_COMMON_API -bool aws_promise_wait_for(struct aws_promise *promise, size_t nanoseconds); - -/* - * Completes the promise and stores the result along with an optional destructor. If the value - * is not taken via `aws_promise_take_value`, it will be destroyed when the promise's reference - * count reaches zero. - * NOTE: Promise cannot be completed twice - */ -AWS_COMMON_API -void aws_promise_complete(struct aws_promise *promise, void *value, void (*dtor)(void *)); - -/* - * Completes the promise and stores the error code - * NOTE: Promise cannot be completed twice - */ -AWS_COMMON_API -void aws_promise_fail(struct aws_promise *promise, int error_code); - -/* - * Returns whether or not the promise has completed (regardless of success or failure) - */ -AWS_COMMON_API -bool aws_promise_is_complete(struct aws_promise *promise); - -/* - * Returns the error code recorded if the promise failed, or 0 if it succeeded - * NOTE: It is fatal to attempt to retrieve the error code before the promise is completed - */ -AWS_COMMON_API -int aws_promise_error_code(struct aws_promise *promise); - -/* - * Returns the value provided to the promise if it succeeded, or NULL if none was provided - * or the promise failed. Check `aws_promise_error_code` to be sure. - * NOTE: The ownership of the value is retained by the promise. - * NOTE: It is fatal to attempt to retrieve the value before the promise is completed - */ -AWS_COMMON_API -void *aws_promise_value(struct aws_promise *promise); - -/* - * Returns the value provided to the promise if it succeeded, or NULL if none was provided - * or the promise failed. Check `aws_promise_error_code` to be sure. - * NOTE: The promise relinquishes ownership of the value, the caller is now responsible for - * freeing any resources associated with the value - * NOTE: It is fatal to attempt to take the value before the promise is completed - */ -AWS_COMMON_API -void *aws_promise_take_value(struct aws_promise *promise); - -AWS_EXTERN_C_END -AWS_POP_SANE_WARNING_LEVEL - -#endif // AWS_COMMON_PROMISE_H diff --git a/source/promise.c b/source/promise.c deleted file mode 100644 index 7c8572457..000000000 --- a/source/promise.c +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -#include -#include -#include -#include - -struct aws_promise { - struct aws_allocator *allocator; - struct aws_mutex mutex; - struct aws_condition_variable cv; - struct aws_ref_count rc; - bool complete; - int error_code; - void *value; - - /* destructor for value, will be invoked if the value is not taken */ - void (*dtor)(void *); -}; - -static void s_aws_promise_dtor(void *ptr) { - struct aws_promise *promise = ptr; - aws_condition_variable_clean_up(&promise->cv); - aws_mutex_clean_up(&promise->mutex); - if (promise->value && promise->dtor) { - promise->dtor(promise->value); - } - aws_mem_release(promise->allocator, promise); -} - -struct aws_promise *aws_promise_new(struct aws_allocator *allocator) { - struct aws_promise *promise = aws_mem_calloc(allocator, 1, sizeof(struct aws_promise)); - promise->allocator = allocator; - aws_ref_count_init(&promise->rc, promise, s_aws_promise_dtor); - aws_mutex_init(&promise->mutex); - aws_condition_variable_init(&promise->cv); - return promise; -} - -struct aws_promise *aws_promise_acquire(struct aws_promise *promise) { - aws_ref_count_acquire(&promise->rc); - return promise; -} - -void aws_promise_release(struct aws_promise *promise) { - aws_ref_count_release(&promise->rc); -} - -static bool s_promise_completed(void *user_data) { - struct aws_promise *promise = user_data; - return promise->complete; -} - -void aws_promise_wait(struct aws_promise *promise) { - aws_mutex_lock(&promise->mutex); - aws_condition_variable_wait_pred(&promise->cv, &promise->mutex, s_promise_completed, promise); - aws_mutex_unlock(&promise->mutex); -} - -bool aws_promise_wait_for(struct aws_promise *promise, size_t nanoseconds) { - aws_mutex_lock(&promise->mutex); - aws_condition_variable_wait_for_pred( - &promise->cv, &promise->mutex, (int64_t)nanoseconds, s_promise_completed, promise); - const bool complete = promise->complete; - aws_mutex_unlock(&promise->mutex); - return complete; -} - -bool aws_promise_is_complete(struct aws_promise *promise) { - aws_mutex_lock(&promise->mutex); - const bool complete = promise->complete; - aws_mutex_unlock(&promise->mutex); - return complete; -} - -void aws_promise_complete(struct aws_promise *promise, void *value, void (*dtor)(void *)) { - aws_mutex_lock(&promise->mutex); - AWS_FATAL_ASSERT(!promise->complete && "aws_promise_complete: cannot complete a promise more than once"); - promise->value = value; - promise->dtor = dtor; - promise->complete = true; - /* Notify before unlocking to prevent a race condition where the recipient spuriously - * awakens after the unlock, sees a fulfilled promise, and attempts to free its resources - * before the notification has actually occured. */ - aws_condition_variable_notify_all(&promise->cv); - aws_mutex_unlock(&promise->mutex); -} - -void aws_promise_fail(struct aws_promise *promise, int error_code) { - AWS_FATAL_ASSERT(error_code != 0 && "aws_promise_fail: cannot fail a promise with a 0 error_code"); - aws_mutex_lock(&promise->mutex); - AWS_FATAL_ASSERT(!promise->complete && "aws_promise_fail: cannot complete a promise more than once"); - promise->error_code = error_code; - promise->complete = true; - aws_condition_variable_notify_all(&promise->cv); - aws_mutex_unlock(&promise->mutex); -} - -int aws_promise_error_code(struct aws_promise *promise) { - AWS_FATAL_ASSERT(aws_promise_is_complete(promise)); - return promise->error_code; -} - -void *aws_promise_value(struct aws_promise *promise) { - AWS_FATAL_ASSERT(aws_promise_is_complete(promise)); - return promise->value; -} - -void *aws_promise_take_value(struct aws_promise *promise) { - AWS_FATAL_ASSERT(aws_promise_is_complete(promise)); - void *value = promise->value; - promise->value = NULL; - promise->dtor = NULL; - return value; -} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6f224e4f2..ee281ac28 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -489,12 +489,6 @@ add_test_case(test_normalize_posix_directory_separator) add_test_case(test_normalize_windows_directory_separator) add_test_case(test_byte_buf_init_from_file) -add_test_case(promise_test_wait_forever) -add_test_case(promise_test_wait_for_a_bit) -add_test_case(promise_test_finish_immediately) -add_test_case(promise_test_finish_before_wait) -add_test_case(promise_test_multiple_waiters) - add_test_case(test_json_parse_from_string) add_test_case(test_json_parse_to_string) diff --git a/tests/promise_test.c b/tests/promise_test.c deleted file mode 100644 index 02f55ec85..000000000 --- a/tests/promise_test.c +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -#include -#include -#include - -#include - -struct promise_test_work { - struct aws_allocator *allocator; - struct aws_promise *promise; - uint64_t work_time; - int error_code; - void *value; - void (*dtor)(void *); -}; - -static void s_promise_test_worker(void *data) { - struct promise_test_work *work = data; - aws_promise_acquire(work->promise); - aws_thread_current_sleep(work->work_time); - if (work->error_code) { - aws_promise_fail(work->promise, work->error_code); - } else { - aws_promise_complete(work->promise, work->value, work->dtor); - } - aws_promise_release(work->promise); -} - -static struct aws_thread s_promise_test_launch_worker(struct promise_test_work *work) { - const struct aws_thread_options *thread_options = aws_default_thread_options(); - AWS_FATAL_ASSERT(thread_options); - struct aws_thread worker_thread; - AWS_FATAL_ASSERT(aws_thread_init(&worker_thread, work->allocator) == AWS_OP_SUCCESS); - AWS_FATAL_ASSERT(aws_thread_launch(&worker_thread, s_promise_test_worker, work, thread_options) == AWS_OP_SUCCESS); - return worker_thread; -} - -struct pmr_payload { - struct aws_allocator *allocator; -}; - -void s_promise_test_free(void *ptr) { - struct pmr_payload *payload = ptr; - aws_mem_release(payload->allocator, payload); -} - -static int s_promise_test_wait_forever(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - struct aws_promise *promise = aws_promise_new(allocator); - ASSERT_NOT_NULL(promise); - - struct pmr_payload *payload = aws_mem_acquire(allocator, 42); - payload->allocator = allocator; - - struct promise_test_work work = { - .allocator = allocator, - .promise = promise, - .work_time = 2 * 1000 * 1000, - .value = payload, - .dtor = s_promise_test_free, - }; - struct aws_thread worker_thread = s_promise_test_launch_worker(&work); - - aws_promise_wait(promise); - ASSERT_SUCCESS(aws_thread_join(&worker_thread)); - aws_promise_release(promise); - - return 0; -} - -AWS_TEST_CASE(promise_test_wait_forever, s_promise_test_wait_forever) - -static int s_promise_test_wait_for_a_bit(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - struct aws_promise *promise = aws_promise_new(allocator); - ASSERT_NOT_NULL(promise); - - struct promise_test_work work = { - .allocator = allocator, - .promise = promise, - .work_time = 3 * 1000 * 1000, - }; - struct aws_thread worker_thread = s_promise_test_launch_worker(&work); - /* wait until the worker finishes, in 500ms intervals */ - while (!aws_promise_wait_for(promise, 500)) - ; - - ASSERT_TRUE(aws_promise_error_code(promise) == 0); - ASSERT_NULL(aws_promise_value(promise)); - - ASSERT_SUCCESS(aws_thread_join(&worker_thread)); - aws_promise_release(promise); - - return 0; -} - -AWS_TEST_CASE(promise_test_wait_for_a_bit, s_promise_test_wait_for_a_bit) - -static int s_promise_test_finish_immediately(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - struct aws_promise *promise = aws_promise_new(allocator); - ASSERT_NOT_NULL(promise); - - struct promise_test_work work = { - .allocator = allocator, - .promise = promise, - .work_time = 0, - }; - struct aws_thread worker_thread = s_promise_test_launch_worker(&work); - aws_promise_wait(promise); - ASSERT_TRUE(aws_promise_error_code(promise) == 0); - ASSERT_NULL(aws_promise_value(promise)); - aws_promise_release(promise); - ASSERT_SUCCESS(aws_thread_join(&worker_thread)); - - return 0; -} - -AWS_TEST_CASE(promise_test_finish_immediately, s_promise_test_finish_immediately) - -static int s_promise_test_finish_before_wait(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - struct aws_promise *promise = aws_promise_new(allocator); - ASSERT_NOT_NULL(promise); - - aws_promise_fail(promise, 1024); - aws_promise_wait(promise); - - ASSERT_TRUE(aws_promise_error_code(promise) == 1024); - ASSERT_NULL(aws_promise_value(promise)); - aws_promise_release(promise); - - return 0; -} - -AWS_TEST_CASE(promise_test_finish_before_wait, s_promise_test_finish_before_wait) - -void s_promise_test_waiter(void *data) { - struct promise_test_work *work = data; - aws_promise_acquire(work->promise); - /* sleep 0.2 seconds */ - aws_thread_current_sleep(1000 * 1000 * 2); - aws_promise_wait(work->promise); - AWS_FATAL_ASSERT(aws_promise_error_code(work->promise) == 0); - aws_promise_release(work->promise); -} - -static int s_promise_test_multiple_waiters(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - struct aws_promise *promise = aws_promise_new(allocator); - ASSERT_NOT_NULL(promise); - - struct promise_test_work work = { - .allocator = allocator, - .promise = promise, - .work_time = 2 * 1000 * 1000, - .value = promise, - }; - struct aws_thread threads[8]; - const struct aws_thread_options *worker_options = aws_default_thread_options(); - for (int idx = 0; idx < AWS_ARRAY_SIZE(threads); ++idx) { - aws_thread_init(&threads[idx], allocator); - aws_thread_launch(&threads[idx], s_promise_test_waiter, &work, worker_options); - } - - aws_thread_current_sleep(1000 * 1000 * 4); - aws_promise_complete(promise, promise, NULL); - aws_promise_release(promise); - - for (int idx = 0; idx < AWS_ARRAY_SIZE(threads); ++idx) { - ASSERT_SUCCESS(aws_thread_join(&threads[idx])); - } - - return 0; -} - -AWS_TEST_CASE(promise_test_multiple_waiters, s_promise_test_multiple_waiters) From 24e23967895201639d7781651b217c8726977d1b Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 8 May 2024 16:18:39 -0700 Subject: [PATCH 19/42] Fix host parsing for IPv6 URI (#1112) --- source/uri.c | 21 +++++++++++++++------ tests/uri_test.c | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/source/uri.c b/source/uri.c index f9ecc9d77..d2d6f77a1 100644 --- a/source/uri.c +++ b/source/uri.c @@ -364,7 +364,9 @@ static void s_parse_authority(struct uri_parser *parser, struct aws_byte_cursor * IPv6 literals and only search for port delimiter after closing bracket.*/ const uint8_t *port_search_start = authority_parse_csr.ptr; size_t port_search_len = authority_parse_csr.len; + bool is_IPv6_literal = false; if (authority_parse_csr.len > 0 && authority_parse_csr.ptr[0] == '[') { + is_IPv6_literal = true; port_search_start = memchr(authority_parse_csr.ptr, ']', authority_parse_csr.len); if (!port_search_start) { parser->state = ERROR; @@ -375,17 +377,24 @@ static void s_parse_authority(struct uri_parser *parser, struct aws_byte_cursor } const uint8_t *port_delim = memchr(port_search_start, ':', port_search_len); - + /* + * RFC-3986 section 3.2.2: A host identified by an IPv6 literal address is represented inside square + * brackets. + * Ignore the square brackets. + */ + parser->uri->host_name = authority_parse_csr; + if (is_IPv6_literal) { + aws_byte_cursor_advance(&parser->uri->host_name, 1); + parser->uri->host_name.len--; + } if (!port_delim) { parser->uri->port = 0; - parser->uri->host_name = authority_parse_csr; return; } - parser->uri->host_name.ptr = authority_parse_csr.ptr; - parser->uri->host_name.len = port_delim - authority_parse_csr.ptr; - - size_t port_len = authority_parse_csr.len - parser->uri->host_name.len - 1; + size_t host_name_length_correction = is_IPv6_literal ? 2 : 0; + parser->uri->host_name.len = port_delim - authority_parse_csr.ptr - host_name_length_correction; + size_t port_len = authority_parse_csr.len - parser->uri->host_name.len - 1 - host_name_length_correction; port_delim += 1; uint64_t port_u64 = 0; diff --git a/tests/uri_test.c b/tests/uri_test.c index 293f6a822..410ff411d 100644 --- a/tests/uri_test.c +++ b/tests/uri_test.c @@ -509,7 +509,7 @@ static int s_test_uri_ipv6_parse(struct aws_allocator *allocator, void *ctx) { aws_byte_cursor_from_c_str("[2001:db8:85a3:8d3:1319:8a2e:370:7348%25en0]:443"); ASSERT_BIN_ARRAYS_EQUALS(expected_authority.ptr, expected_authority.len, uri.authority.ptr, uri.authority.len); - struct aws_byte_cursor expected_host = aws_byte_cursor_from_c_str("[2001:db8:85a3:8d3:1319:8a2e:370:7348%25en0]"); + struct aws_byte_cursor expected_host = aws_byte_cursor_from_c_str("2001:db8:85a3:8d3:1319:8a2e:370:7348%25en0"); ASSERT_BIN_ARRAYS_EQUALS(expected_host.ptr, expected_host.len, uri.host_name.ptr, uri.host_name.len); ASSERT_UINT_EQUALS(443, uri.port); @@ -540,7 +540,7 @@ static int s_test_uri_ipv6_no_port_parse(struct aws_allocator *allocator, void * aws_byte_cursor_from_c_str("[2001:db8:85a3:8d3:1319:8a2e:370:7348%25en0]"); ASSERT_BIN_ARRAYS_EQUALS(expected_authority.ptr, expected_authority.len, uri.authority.ptr, uri.authority.len); - struct aws_byte_cursor expected_host = aws_byte_cursor_from_c_str("[2001:db8:85a3:8d3:1319:8a2e:370:7348%25en0]"); + struct aws_byte_cursor expected_host = aws_byte_cursor_from_c_str("2001:db8:85a3:8d3:1319:8a2e:370:7348%25en0"); ASSERT_BIN_ARRAYS_EQUALS(expected_host.ptr, expected_host.len, uri.host_name.ptr, uri.host_name.len); struct aws_byte_cursor expected_path = aws_byte_cursor_from_c_str(""); From 36a716eed3da79a74460f6c1e8e9b6a119399962 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Fri, 10 May 2024 16:22:14 -0700 Subject: [PATCH 20/42] Fix aws_host_utils_is_ipv6 function (#1114) --- source/host_utils.c | 15 +-------------- tests/host_util_test.c | 15 +++++++-------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/source/host_utils.c b/source/host_utils.c index 6cd26ba88..f20f3dd5e 100644 --- a/source/host_utils.c +++ b/source/host_utils.c @@ -20,10 +20,6 @@ static bool s_is_ipv6_char(uint8_t value) { return aws_isxdigit(value) || value == ':'; } -static bool s_starts_with(struct aws_byte_cursor cur, uint8_t ch) { - return cur.len > 0 && cur.ptr[0] == ch; -} - static bool s_ends_with(struct aws_byte_cursor cur, uint8_t ch) { return cur.len > 0 && cur.ptr[cur.len - 1] == ch; } @@ -69,8 +65,7 @@ static struct aws_byte_cursor s_percent_uri_enc = AWS_BYTE_CUR_INIT_FROM_STRING_ * ipv6 literal can be scoped by to zone by appending % followed by zone name * ( does not look like there is length reqs on zone name length. this * implementation enforces that its > 1 ) - * ipv6 can be embedded in url, in which case it must be wrapped inside [] - * and % be uri encoded as %25. + * ipv6 can be embedded in url, in which case % must be uri encoded as %25. * Implementation is fairly trivial and just iterates through the string * keeping track of the spec above. */ @@ -79,14 +74,6 @@ bool aws_host_utils_is_ipv6(struct aws_byte_cursor host, bool is_uri_encoded) { return false; } - if (is_uri_encoded) { - if (!s_starts_with(host, '[') || !s_ends_with(host, ']')) { - return false; - } - aws_byte_cursor_advance(&host, 1); - --host.len; - } - struct aws_byte_cursor substr = {0}; /* first split is required ipv6 part */ bool is_split = aws_byte_cursor_next_split(&host, '%', &substr); diff --git a/tests/host_util_test.c b/tests/host_util_test.c index bf0192610..1f35f8b7f 100644 --- a/tests/host_util_test.c +++ b/tests/host_util_test.c @@ -46,10 +46,10 @@ static int s_test_is_ipv6(struct aws_allocator *allocator, void *ctx) { ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("0:0:0:0:0:0:0:1"), false)); ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fd00:ec2::23"), false)); ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fd00:ec2:0:0:0:0:0:23"), false)); - ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[2001:0db8:0000:0000:0000:8a2e:0370:7334]"), true)); - ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1]"), true)); - ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1%25en0]"), true)); - ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[2001:db8:85a3:8d3:1319:8a2e:370:7348]"), true)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("2001:0db8:0000:0000:0000:8a2e:0370:7334"), true)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1"), true)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1%25en0"), true)); + ASSERT_TRUE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("2001:db8:85a3:8d3:1319:8a2e:370:7348"), true)); ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("2001:0db8:0000:0000:0000:8a2e:0370"), false)); ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("2001:0db8:0000:0000:0000:8a2e:0370:"), false)); @@ -63,12 +63,11 @@ static int s_test_is_ipv6(struct aws_allocator *allocator, void *ctx) { ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("z001::8a2e::8745"), false)); ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("::2001:0db8:0000:0000:8a2e:0370:7334"), false)); - ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1%25en0"), true)); - ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1%en0]"), true)); - ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1%24en0]"), true)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1%en0"), true)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1%24en0"), true)); ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1%25en0"), true)); ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1%25en0]"), true)); - ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("[fe80::1%25]"), true)); + ASSERT_FALSE(aws_host_utils_is_ipv6(aws_byte_cursor_from_c_str("fe80::1%25"), true)); return AWS_OP_SUCCESS; } From 6ebf3bc45ffcaa3709a8bab8dc373d82e7cdd4cc Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Tue, 14 May 2024 10:12:40 -0700 Subject: [PATCH 21/42] Task scheduler: log at TRACE, instead of DEBUG (#1115) --- source/task_scheduler.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/task_scheduler.c b/source/task_scheduler.c index bca150e39..9076ae249 100644 --- a/source/task_scheduler.c +++ b/source/task_scheduler.c @@ -33,7 +33,7 @@ const char *aws_task_status_to_c_str(enum aws_task_status status) { void aws_task_run(struct aws_task *task, enum aws_task_status status) { AWS_ASSERT(task->fn); - AWS_LOGF_DEBUG( + AWS_LOGF_TRACE( AWS_LS_COMMON_TASK_SCHEDULER, "id=%p: Running %s task with %s status", (void *)task, @@ -129,7 +129,7 @@ void aws_task_scheduler_schedule_now(struct aws_task_scheduler *scheduler, struc AWS_ASSERT(task); AWS_ASSERT(task->fn); - AWS_LOGF_DEBUG( + AWS_LOGF_TRACE( AWS_LS_COMMON_TASK_SCHEDULER, "id=%p: Scheduling %s task for immediate execution", (void *)task, @@ -152,7 +152,7 @@ void aws_task_scheduler_schedule_future( AWS_ASSERT(task); AWS_ASSERT(task->fn); - AWS_LOGF_DEBUG( + AWS_LOGF_TRACE( AWS_LS_COMMON_TASK_SCHEDULER, "id=%p: Scheduling %s task for future execution at time %" PRIu64, (void *)task, From ab2cfa973af862560ec71a3de313c6033db996f9 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 20 May 2024 14:47:24 -0700 Subject: [PATCH 22/42] Update CJson to v1.7.18 (#1116) Co-authored-by: Dengke Tang --- .github/workflows/ci.yml | 2 +- source/external/cJSON.c | 20 +++++++++++++++++--- source/external/cJSON.h | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b25b05cff..eff75483f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,7 @@ jobs: python builder.pyz build -p ${{ env.PACKAGE_NAME }} --target windows-${{ matrix.arch }} --compiler msvc-16 windows-vc15: - runs-on: windows-2022 # latest + runs-on: windows-2019 # windows-2019 is last env with Visual Studio 2017 (v15.0) strategy: matrix: arch: [x86, x64] diff --git a/source/external/cJSON.c b/source/external/cJSON.c index 9ca6351bf..8eac43afd 100644 --- a/source/external/cJSON.c +++ b/source/external/cJSON.c @@ -127,7 +127,7 @@ CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) } /* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ -#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 17) +#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 18) #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. #endif @@ -273,10 +273,12 @@ CJSON_PUBLIC(void) cJSON_Delete(cJSON *item) if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL)) { global_hooks.deallocate(item->valuestring); + item->valuestring = NULL; } if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) { global_hooks.deallocate(item->string); + item->string = NULL; } global_hooks.deallocate(item); item = next; @@ -407,6 +409,7 @@ CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) return object->valuedouble = number; } +/* Note: when passing a NULL valuestring, cJSON_SetValuestring treats this as an error and return NULL */ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) { char *copy = NULL; @@ -415,8 +418,8 @@ CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) { return NULL; } - /* return NULL if the object is corrupted */ - if (object->valuestring == NULL) + /* return NULL if the object is corrupted or valuestring is NULL */ + if (object->valuestring == NULL || valuestring == NULL) { return NULL; } @@ -903,6 +906,7 @@ static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_bu if (output != NULL) { input_buffer->hooks.deallocate(output); + output = NULL; } if (input_pointer != NULL) @@ -1249,6 +1253,7 @@ static unsigned char *print(const cJSON * const item, cJSON_bool format, const i /* free the buffer */ hooks->deallocate(buffer->buffer); + buffer->buffer = NULL; } return printed; @@ -1257,11 +1262,13 @@ static unsigned char *print(const cJSON * const item, cJSON_bool format, const i if (buffer->buffer != NULL) { hooks->deallocate(buffer->buffer); + buffer->buffer = NULL; } if (printed != NULL) { hooks->deallocate(printed); + printed = NULL; } return NULL; @@ -1302,6 +1309,7 @@ CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON if (!print_value(item, &p)) { global_hooks.deallocate(p.buffer); + p.buffer = NULL; return NULL; } @@ -1673,6 +1681,11 @@ static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_bu current_item = new_item; } + if (cannot_access_at_index(input_buffer, 1)) + { + goto fail; /* nothing comes after the comma */ + } + /* parse the name of the child */ input_buffer->offset++; buffer_skip_whitespace(input_buffer); @@ -3140,6 +3153,7 @@ CJSON_PUBLIC(void *) cJSON_malloc(size_t size) CJSON_PUBLIC(void) cJSON_free(void *object) { global_hooks.deallocate(object); + object = NULL; } /* Amazon edit */ /* NOLINTEND */ diff --git a/source/external/cJSON.h b/source/external/cJSON.h index 47594475e..48cdda9ac 100644 --- a/source/external/cJSON.h +++ b/source/external/cJSON.h @@ -88,7 +88,7 @@ then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJ /* project version */ #define CJSON_VERSION_MAJOR 1 #define CJSON_VERSION_MINOR 7 -#define CJSON_VERSION_PATCH 17 +#define CJSON_VERSION_PATCH 18 #include From ce899b907b8218a6d2c46f30bd7a37f94909d502 Mon Sep 17 00:00:00 2001 From: Qinheping Hu Date: Mon, 20 May 2024 17:39:43 -0500 Subject: [PATCH 23/42] avoid overflowing pointers passed to memcpy (#874) Co-authored-by: Felipe R. Monteiro --- source/array_list.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/array_list.c b/source/array_list.c index b6ea5a090..a890115e9 100644 --- a/source/array_list.c +++ b/source/array_list.c @@ -184,9 +184,12 @@ static void aws_array_list_mem_swap(void *AWS_RESTRICT item1, void *AWS_RESTRICT } size_t remainder = item_size & (SLICE - 1); /* item_size % SLICE */ - memcpy((void *)temp, (void *)item1, remainder); - memcpy((void *)item1, (void *)item2, remainder); - memcpy((void *)item2, (void *)temp, remainder); + + if (remainder) { + memcpy((void *)temp, (void *)item1, remainder); + memcpy((void *)item1, (void *)item2, remainder); + memcpy((void *)item2, (void *)temp, remainder); + } } void aws_array_list_swap(struct aws_array_list *AWS_RESTRICT list, size_t a, size_t b) { From dae0e52ffd53aa7d14157b9d97458892d190023c Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Thu, 30 May 2024 10:05:33 -0700 Subject: [PATCH 24/42] Fix test skipping (#1118) **Issue:** The SKIP code worked in `aws-crt-cpp`'s tests, but not in any `aws-c-*` tests **Description of changes:** - Set SKIP_RETURN_CODE in CMake script that generates C test runner (the CPP script had it, but not the C script) - Skip leak checks when a test is skipped - this keeps the tests simple, you can bail out in the middle without cleaning up --- cmake/AwsTestHarness.cmake | 1 + include/aws/testing/aws_test_harness.h | 28 ++++++++------------------ tests/thread_test.c | 2 +- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/cmake/AwsTestHarness.cmake b/cmake/AwsTestHarness.cmake index 84435b35f..484b66635 100644 --- a/cmake/AwsTestHarness.cmake +++ b/cmake/AwsTestHarness.cmake @@ -59,6 +59,7 @@ function(generate_test_driver driver_exe_name) foreach(name IN LISTS TEST_CASES) add_test(${name} ${driver_exe_name} "${name}") + set_tests_properties("${name}" PROPERTIES SKIP_RETURN_CODE ${SKIP_RETURN_CODE_VALUE}) endforeach() # Clear test cases in case another driver needs to be generated diff --git a/include/aws/testing/aws_test_harness.h b/include/aws/testing/aws_test_harness.h index 3c18e1ab0..2f62bab62 100644 --- a/include/aws/testing/aws_test_harness.h +++ b/include/aws/testing/aws_test_harness.h @@ -83,18 +83,6 @@ static int s_cunit_failure_message0( * that may be returned from various tools (e.g. sanitizer). */ #define SKIP (103) -#define POSTSKIP_INTERNAL() \ - do { \ - return SKIP; \ - } while (0) - -#define RETURN_SKIP(format, ...) \ - do { \ - printf(format, ##__VA_ARGS__); \ - printf("\n"); \ - POSTSKIP_INTERNAL(); \ - } while (0) - #define RETURN_SUCCESS(format, ...) \ do { \ printf(format, ##__VA_ARGS__); \ @@ -468,7 +456,7 @@ static inline int s_aws_run_test_case(struct aws_test_harness *harness) { test_res |= harness->on_after(allocator, setup_res, harness->ctx); } - if (test_res != AWS_OP_SUCCESS && test_res != AWS_OP_SKIP) { + if (test_res != AWS_OP_SUCCESS) { goto fail; } @@ -492,21 +480,21 @@ static inline int s_aws_run_test_case(struct aws_test_harness *harness) { aws_logger_set(NULL); aws_logger_clean_up(&err_logger); - if (test_res == AWS_OP_SUCCESS) { - RETURN_SUCCESS("%s [ \033[32mOK\033[0m ]", harness->test_name); - } else if (test_res == AWS_OP_SKIP) { - RETURN_SKIP("%s [ \033[32mSKIP\033[0m ]", harness->test_name); - } + RETURN_SUCCESS("%s [ \033[32mOK\033[0m ]", harness->test_name); fail: - PRINT_FAIL_WITHOUT_LOCATION("%s [ \033[31mFAILED\033[0m ]", harness->test_name); + if (test_res == AWS_OP_SKIP) { + fprintf(AWS_TESTING_REPORT_FD, "%s [ \033[32mSKIP\033[0m ]\n", harness->test_name); + } else { + PRINT_FAIL_WITHOUT_LOCATION("%s [ \033[31mFAILED\033[0m ]", harness->test_name); + } /* Use _Exit() to terminate without cleaning up resources. * This prevents LeakSanitizer spam (yes, we know failing tests don't bother cleaning up). * It also prevents errors where threads that haven't cleaned are still using the logger declared in this fn. */ fflush(AWS_TESTING_REPORT_FD); fflush(stdout); fflush(stderr); - _Exit(FAILURE); + _Exit(test_res == AWS_OP_SKIP ? SKIP : FAILURE); } /* Enables terminal escape sequences for text coloring on Windows. */ diff --git a/tests/thread_test.c b/tests/thread_test.c index 9d9fb62d9..1b1582131 100644 --- a/tests/thread_test.c +++ b/tests/thread_test.c @@ -243,7 +243,7 @@ static int s_test_managed_thread_join_timeout(struct aws_allocator *allocator, v /* * Increase the timeout and shut down */ - aws_thread_set_managed_join_timeout_ns(aws_timestamp_convert(5, AWS_TIMESTAMP_SECS, AWS_TIMESTAMP_NANOS, NULL)); + aws_thread_set_managed_join_timeout_ns(aws_timestamp_convert(10, AWS_TIMESTAMP_SECS, AWS_TIMESTAMP_NANOS, NULL)); aws_common_library_clean_up(); From 24ac711499116ffbb957f2eac0a996f7e205d838 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Mon, 3 Jun 2024 13:22:39 -0700 Subject: [PATCH 25/42] Fix CMake Windows ARM typo (#1119) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d6e615a3..30826bdbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,7 +152,7 @@ if (USE_CPU_EXTENSIONS) ) endif() elseif (AWS_ARCH_ARM64 OR AWS_ARCH_ARM32) - if (WINDOWS) + if (WIN32) file(GLOB AWS_COMMON_ARCH_SRC "source/arch/arm/windows/*.c" ) From 06cf4d82adad52723305dc2aeb06bb6439b5df7f Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Mon, 3 Jun 2024 16:59:27 -0700 Subject: [PATCH 26/42] Fix weird decimal formatting in tests (#1121) When we switched test harness to print errors as decimals in PR https://github.com/awslabs/aws-c-common/pull/1100, we forgot to remove some of the hex decorations --- include/aws/testing/aws_test_harness.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/aws/testing/aws_test_harness.h b/include/aws/testing/aws_test_harness.h index 2f62bab62..769e75fc9 100644 --- a/include/aws/testing/aws_test_harness.h +++ b/include/aws/testing/aws_test_harness.h @@ -135,7 +135,7 @@ static int s_cunit_failure_message0( if (assert_rv != AWS_OP_SUCCESS) { \ if (!PRINT_FAIL_INTERNAL0(__VA_ARGS__)) { \ PRINT_FAIL_INTERNAL0( \ - "Expected success at %s; got return value %d with last error 0x%04d\n", \ + "Expected success at %s; got return value %d with last error %d\n", \ #condition, \ assert_rv, \ aws_last_error()); \ @@ -150,7 +150,7 @@ static int s_cunit_failure_message0( if (assert_rv != AWS_OP_ERR) { \ if (!PRINT_FAIL_INTERNAL0(__VA_ARGS__)) { \ PRINT_FAIL_INTERNAL0( \ - "Expected failure at %s; got return value %d with last error 0x%04d\n", \ + "Expected failure at %s; got return value %d with last error %d\n", \ #condition, \ assert_rv, \ aws_last_error()); \ @@ -167,7 +167,7 @@ static int s_cunit_failure_message0( if (assert_rv != AWS_OP_ERR) { \ fprintf( \ AWS_TESTING_REPORT_FD, \ - "%sExpected error but no error occurred; rv=%d, aws_last_error=%04d (expected %04d): ", \ + "%sExpected error but no error occurred; rv=%d, aws_last_error=%d (expected %d): ", \ FAIL_PREFIX, \ assert_rv, \ assert_err, \ @@ -180,7 +180,7 @@ static int s_cunit_failure_message0( if (assert_err != assert_err_expect) { \ fprintf( \ AWS_TESTING_REPORT_FD, \ - "%sIncorrect error code; aws_last_error=%04d (expected %04d): ", \ + "%sIncorrect error code; aws_last_error=%d (expected %d): ", \ FAIL_PREFIX, \ assert_err, \ assert_err_expect); \ From d2d5da422885030c0edf9603bab4a018caf4e60c Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Wed, 5 Jun 2024 12:56:57 -0700 Subject: [PATCH 27/42] Fix memtracer bad assumptions on the size of stack trace (#1122) --- source/memtrace.c | 59 +++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/source/memtrace.c b/source/memtrace.c index fbdcf86df..ee1f291d7 100644 --- a/source/memtrace.c +++ b/source/memtrace.c @@ -131,29 +131,43 @@ static void s_alloc_tracer_track(struct alloc_tracer *tracer, void *ptr, size_t aws_high_res_clock_get_ticks(&alloc->time); if (tracer->level == AWS_MEMTRACE_STACKS) { - /* capture stack frames, skip 2 for this function and the allocation vtable function */ + /* capture stack frames, + * skip 2 for this function and the allocation vtable function if we have a full stack trace + * and otherwise just capture what ever stack trace we got + */ AWS_VARIABLE_LENGTH_ARRAY(void *, stack_frames, (FRAMES_TO_SKIP + tracer->frames_per_stack)); size_t stack_depth = aws_backtrace(stack_frames, FRAMES_TO_SKIP + tracer->frames_per_stack); - if (stack_depth >= FRAMES_TO_SKIP) { - /* hash the stack pointers */ - struct aws_byte_cursor stack_cursor = - aws_byte_cursor_from_array(stack_frames, stack_depth * sizeof(void *)); - uint64_t stack_id = aws_hash_byte_cursor_ptr(&stack_cursor); - alloc->stack = stack_id; /* associate the stack with the alloc */ - - aws_mutex_lock(&tracer->mutex); - struct aws_hash_element *item = NULL; - int was_created = 0; - AWS_FATAL_ASSERT( - AWS_OP_SUCCESS == - aws_hash_table_create(&tracer->stacks, (void *)(uintptr_t)stack_id, &item, &was_created)); - /* If this is a new stack, save it to the hash */ - if (was_created) { - struct stack_trace *stack = aws_mem_calloc( - aws_default_allocator(), - 1, - sizeof(struct stack_trace) + (sizeof(void *) * tracer->frames_per_stack)); - AWS_FATAL_ASSERT(stack); + AWS_FATAL_ASSERT(stack_depth > 0); + + /* hash the stack pointers */ + struct aws_byte_cursor stack_cursor = aws_byte_cursor_from_array(stack_frames, stack_depth * sizeof(void *)); + uint64_t stack_id = aws_hash_byte_cursor_ptr(&stack_cursor); + alloc->stack = stack_id; /* associate the stack with the alloc */ + + aws_mutex_lock(&tracer->mutex); + struct aws_hash_element *item = NULL; + int was_created = 0; + AWS_FATAL_ASSERT( + AWS_OP_SUCCESS == aws_hash_table_create(&tracer->stacks, (void *)(uintptr_t)stack_id, &item, &was_created)); + /* If this is a new stack, save it to the hash */ + if (was_created) { + struct stack_trace *stack = aws_mem_calloc( + aws_default_allocator(), 1, sizeof(struct stack_trace) + (sizeof(void *) * tracer->frames_per_stack)); + AWS_FATAL_ASSERT(stack); + /** + * Optimizations can affect the number of frames we get and in pathological cases we can + * get fewer than FRAMES_TO_SKIP frames, but always at least 1 because code has to start somewhere. + * (looking at you gcc with -O3 on aarch64) + * With optimizations on we cannot trust the stack trace too much. + * Memtracer makes an assumption that stack trace will be available in all cases if stack trace api + * works. So in the pathological case of stack_depth <= FRAMES_TO_SKIP lets record all the frames we + * have, to at least have an anchor for where allocation is comming from, however inaccurate it is. + */ + if (stack_depth <= FRAMES_TO_SKIP) { + memcpy((void **)&stack->frames[0], &stack_frames[0], (stack_depth) * sizeof(void *)); + stack->depth = stack_depth; + item->value = stack; + } else { memcpy( (void **)&stack->frames[0], &stack_frames[FRAMES_TO_SKIP], @@ -161,8 +175,9 @@ static void s_alloc_tracer_track(struct alloc_tracer *tracer, void *ptr, size_t stack->depth = stack_depth - FRAMES_TO_SKIP; item->value = stack; } - aws_mutex_unlock(&tracer->mutex); } + + aws_mutex_unlock(&tracer->mutex); } aws_mutex_lock(&tracer->mutex); From 4f874cea50a70bc6ebcd85c6ce1c6c0016b5aff4 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 5 Jun 2024 13:53:16 -0700 Subject: [PATCH 28/42] disable optimization was not working (#1123) --- tests/memtrace_test.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/memtrace_test.c b/tests/memtrace_test.c index 474d2a422..b3be1b7e9 100644 --- a/tests/memtrace_test.c +++ b/tests/memtrace_test.c @@ -63,32 +63,36 @@ static int s_test_memtrace_count(struct aws_allocator *allocator, void *ctx) { } AWS_TEST_CASE(test_memtrace_count, s_test_memtrace_count) -#if defined(__GNUC__) || defined(__clang__) -# define AWS_PREVENT_OPTIMIZATION __asm__ __volatile__("" ::: "memory") -#else -# define AWS_PREVENT_OPTIMIZATION +/* Disable optimization for tests to make sure those allocation function still exists. */ +#if defined(__clang__) +# pragma clang optimize off +#elif defined(__GNUC__) || defined(__GNUG__) +# pragma GCC push_options +# pragma GCC optimize("O0") #endif AWS_NO_INLINE void *s_alloc_1(struct aws_allocator *allocator, size_t size) { - AWS_PREVENT_OPTIMIZATION; return aws_mem_acquire(allocator, size); } AWS_NO_INLINE void *s_alloc_2(struct aws_allocator *allocator, size_t size) { - AWS_PREVENT_OPTIMIZATION; return aws_mem_acquire(allocator, size); } AWS_NO_INLINE void *s_alloc_3(struct aws_allocator *allocator, size_t size) { - AWS_PREVENT_OPTIMIZATION; return aws_mem_acquire(allocator, size); } AWS_NO_INLINE void *s_alloc_4(struct aws_allocator *allocator, size_t size) { - AWS_PREVENT_OPTIMIZATION; return aws_mem_acquire(allocator, size); } +#if defined(__clang__) +# pragma clang optimize on +#elif defined(__GNUC__) || defined(__GNUG__) +# pragma GCC pop_options +#endif + static struct aws_logger s_test_logger; static int s_test_memtrace_stacks(struct aws_allocator *allocator, void *ctx) { From 4fccc79c8402b37802d8cb97899464b43dc6bbfc Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Wed, 12 Jun 2024 09:52:32 -0700 Subject: [PATCH 29/42] clang-format 18 (#1113) **Issue:** - We've used clang-format 9 since forever, but it's not easy for people to get such an old version anymore. - You really need everyone on the same exact version clang-format, even PATCH matters - It's very hard to get the same version on everyone's machine (Ubuntu 24 has 18, brew on MacOS has 18, but Amazon Linux 2 only has 11, and AL2023 only has 15) **Description of Changes:** - start using latest version of clang-format: 18.1.6 - use [pipx](https://github.com/pypa/pipx) to install and run [clang-format from PyPI](https://pypi.org/project/clang-format). This lets everyone run the same version, regardless of OS - rewrite ~format-check.sh~ as `format-check.py` - now we can run it on Windows - most people on this team know Python better than Bash - add `-i` option to edit files in place - the script runs the specific clang-format version via pipx, so even if you have some other version of clang-format installed, the script will get it right - CI just runs `./format-check.py` instead of using some 3rdparty Github Action --- .github/workflows/clang-format.yml | 11 +++--- format-check.py | 47 +++++++++++++++++++++++++ format-check.sh | 24 ------------- include/aws/common/atomics.h | 6 ++-- include/aws/common/byte_buf.h | 2 +- include/aws/common/condition_variable.h | 6 ++-- include/aws/common/error.h | 7 ++-- include/aws/common/logging.h | 4 +-- include/aws/common/macros.h | 2 +- include/aws/common/mutex.h | 6 ++-- include/aws/common/rw_lock.h | 6 ++-- include/aws/common/statistics.h | 2 +- include/aws/common/thread.h | 3 +- source/allocator.c | 2 +- source/android/logging.c | 4 ++- source/common.c | 2 +- source/posix/clock.c | 2 +- source/priority_queue.c | 2 +- tests/assert_test.c | 2 ++ tests/condition_variable_test.c | 32 +++++++++-------- tests/error_test.c | 16 +++++---- 21 files changed, 106 insertions(+), 82 deletions(-) create mode 100755 format-check.py delete mode 100755 format-check.sh diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index d914a0955..36388d688 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -5,15 +5,12 @@ on: [push] jobs: clang-format: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-24.04 # latest steps: - name: Checkout Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: clang-format lint - uses: DoozyX/clang-format-lint-action@v0.3.1 - with: - # List of extensions to check - extensions: c,h,inl - source: 'source include tests verification' + run: | + ./format-check.py diff --git a/format-check.py b/format-check.py new file mode 100755 index 000000000..b9e3520cc --- /dev/null +++ b/format-check.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +import argparse +import os +from pathlib import Path +import re +from subprocess import list2cmdline, run +from tempfile import NamedTemporaryFile + +CLANG_FORMAT_VERSION = '18.1.6' + +INCLUDE_REGEX = re.compile( + r'^(include|source|tests|verification)/.*\.(c|h|inl)$') +EXCLUDE_REGEX = re.compile(r'^$') + +arg_parser = argparse.ArgumentParser(description="Check with clang-format") +arg_parser.add_argument('-i', '--inplace-edit', action='store_true', + help="Edit files inplace") +args = arg_parser.parse_args() + +os.chdir(Path(__file__).parent) + +# create file containing list of all files to format +filepaths_file = NamedTemporaryFile(delete=False) +for dirpath, dirnames, filenames in os.walk('.'): + for filename in filenames: + # our regexes expect filepath to use forward slash + filepath = Path(dirpath, filename).as_posix() + if not INCLUDE_REGEX.match(filepath): + continue + if EXCLUDE_REGEX.match(filepath): + continue + + filepaths_file.write(f"{filepath}\n".encode()) +filepaths_file.close() + +# use pipx to run clang-format from PyPI +# this is a simple way to run the same clang-format version regardless of OS +cmd = ['pipx', 'run', f'clang-format=={CLANG_FORMAT_VERSION}', + f'--files={filepaths_file.name}'] +if args.inplace_edit: + cmd += ['-i'] +else: + cmd += ['--Werror', '--dry-run'] + +print(f"{Path.cwd()}$ {list2cmdline(cmd)}") +if run(cmd).returncode: + exit(1) diff --git a/format-check.sh b/format-check.sh deleted file mode 100755 index d292e9d7f..000000000 --- a/format-check.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -if [[ -z $CLANG_FORMAT ]] ; then - CLANG_FORMAT=clang-format -fi - -if ! type $CLANG_FORMAT 2> /dev/null ; then - echo "No appropriate clang-format found." - exit 1 -fi - -FAIL=0 -SOURCE_FILES=`find source include tests verification -type f \( -name '*.h' -o -name '*.c' -o -name '*.inl' \)` -for i in $SOURCE_FILES -do - $CLANG_FORMAT -output-replacements-xml $i | grep -c " /dev/null - if [ $? -ne 1 ] - then - echo "$i failed clang-format check." - FAIL=1 - fi -done - -exit $FAIL diff --git a/include/aws/common/atomics.h b/include/aws/common/atomics.h index ef64f985a..7f5445334 100644 --- a/include/aws/common/atomics.h +++ b/include/aws/common/atomics.h @@ -79,14 +79,12 @@ enum aws_memory_order { /** * Statically initializes an aws_atomic_var to a given size_t value. */ -#define AWS_ATOMIC_INIT_INT(x) \ - { .value = (void *)(uintptr_t)(x) } +#define AWS_ATOMIC_INIT_INT(x) {.value = (void *)(uintptr_t)(x)} /** * Statically initializes an aws_atomic_var to a given void * value. */ -#define AWS_ATOMIC_INIT_PTR(x) \ - { .value = (void *)(x) } +#define AWS_ATOMIC_INIT_PTR(x) {.value = (void *)(x)} AWS_EXTERN_C_BEGIN diff --git a/include/aws/common/byte_buf.h b/include/aws/common/byte_buf.h index 6fc5c3ff9..11efd3f78 100644 --- a/include/aws/common/byte_buf.h +++ b/include/aws/common/byte_buf.h @@ -61,7 +61,7 @@ struct aws_byte_cursor { * Helper Macro for initializing a byte cursor from a string literal */ #define AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL(literal) \ - { .ptr = (uint8_t *)(const char *)(literal), .len = sizeof(literal) - 1 } + {.ptr = (uint8_t *)(const char *)(literal), .len = sizeof(literal) - 1} /** * Signature for function argument to trim APIs diff --git a/include/aws/common/condition_variable.h b/include/aws/common/condition_variable.h index ba5984cb8..82431dd4f 100644 --- a/include/aws/common/condition_variable.h +++ b/include/aws/common/condition_variable.h @@ -37,11 +37,9 @@ struct aws_condition_variable { * CONDITION_VARIABLE_INIT. */ #ifdef _WIN32 -# define AWS_CONDITION_VARIABLE_INIT \ - { .condition_handle = NULL, .initialized = true } +# define AWS_CONDITION_VARIABLE_INIT {.condition_handle = NULL, .initialized = true} #else -# define AWS_CONDITION_VARIABLE_INIT \ - { .condition_handle = PTHREAD_COND_INITIALIZER, .initialized = true } +# define AWS_CONDITION_VARIABLE_INIT {.condition_handle = PTHREAD_COND_INITIALIZER, .initialized = true} #endif AWS_EXTERN_C_BEGIN diff --git a/include/aws/common/error.h b/include/aws/common/error.h index a40b193fa..8c5d45f69 100644 --- a/include/aws/common/error.h +++ b/include/aws/common/error.h @@ -20,7 +20,7 @@ AWS_PUSH_SANE_WARNING_LEVEL /* Each library gets space for 2^^10 error entries */ #define AWS_ERROR_ENUM_STRIDE_BITS 10 #define AWS_ERROR_ENUM_STRIDE (1U << AWS_ERROR_ENUM_STRIDE_BITS) -#define AWS_ERROR_ENUM_BEGIN_RANGE(x) ((x)*AWS_ERROR_ENUM_STRIDE) +#define AWS_ERROR_ENUM_BEGIN_RANGE(x) ((x) * AWS_ERROR_ENUM_STRIDE) #define AWS_ERROR_ENUM_END_RANGE(x) (((x) + 1) * AWS_ERROR_ENUM_STRIDE - 1) struct aws_error_info { @@ -38,7 +38,10 @@ struct aws_error_info_list { #define AWS_DEFINE_ERROR_INFO(C, ES, LN) \ { \ - .literal_name = #C, .error_code = (C), .error_str = (ES), .lib_name = (LN), \ + .literal_name = #C, \ + .error_code = (C), \ + .error_str = (ES), \ + .lib_name = (LN), \ .formatted_name = LN ": " #C ", " ES, \ } diff --git a/include/aws/common/logging.h b/include/aws/common/logging.h index 009996475..f71a375b5 100644 --- a/include/aws/common/logging.h +++ b/include/aws/common/logging.h @@ -62,7 +62,7 @@ enum { AWS_LOG_SUBJECT_STRIDE_BITS = 10, }; #define AWS_LOG_SUBJECT_STRIDE (1U << AWS_LOG_SUBJECT_STRIDE_BITS) -#define AWS_LOG_SUBJECT_BEGIN_RANGE(x) ((x)*AWS_LOG_SUBJECT_STRIDE) +#define AWS_LOG_SUBJECT_BEGIN_RANGE(x) ((x) * AWS_LOG_SUBJECT_STRIDE) #define AWS_LOG_SUBJECT_END_RANGE(x) (((x) + 1) * AWS_LOG_SUBJECT_STRIDE - 1) struct aws_log_subject_info { @@ -72,7 +72,7 @@ struct aws_log_subject_info { }; #define DEFINE_LOG_SUBJECT_INFO(id, name, desc) \ - { .subject_id = (id), .subject_name = (name), .subject_description = (desc) } + {.subject_id = (id), .subject_name = (name), .subject_description = (desc)} struct aws_log_subject_info_list { struct aws_log_subject_info *subject_list; diff --git a/include/aws/common/macros.h b/include/aws/common/macros.h index da3298544..f5c64c6aa 100644 --- a/include/aws/common/macros.h +++ b/include/aws/common/macros.h @@ -160,6 +160,6 @@ enum { AWS_CACHE_LINE = 64 }; * this will get you back to the pointer of the object. member is the name of * the instance of struct aws_linked_list_node in your struct. */ -#define AWS_CONTAINER_OF(ptr, type, member) ((type *)((uint8_t *)(ptr)-offsetof(type, member))) +#define AWS_CONTAINER_OF(ptr, type, member) ((type *)((uint8_t *)(ptr) - offsetof(type, member))) #endif /* AWS_COMMON_MACROS_H */ diff --git a/include/aws/common/mutex.h b/include/aws/common/mutex.h index 3a491abc9..b59a58474 100644 --- a/include/aws/common/mutex.h +++ b/include/aws/common/mutex.h @@ -26,11 +26,9 @@ struct aws_mutex { }; #ifdef _WIN32 -# define AWS_MUTEX_INIT \ - { .mutex_handle = NULL, .initialized = true } +# define AWS_MUTEX_INIT {.mutex_handle = NULL, .initialized = true} #else -# define AWS_MUTEX_INIT \ - { .mutex_handle = PTHREAD_MUTEX_INITIALIZER, .initialized = true } +# define AWS_MUTEX_INIT {.mutex_handle = PTHREAD_MUTEX_INITIALIZER, .initialized = true} #endif AWS_EXTERN_C_BEGIN diff --git a/include/aws/common/rw_lock.h b/include/aws/common/rw_lock.h index 538841f63..3ee778537 100644 --- a/include/aws/common/rw_lock.h +++ b/include/aws/common/rw_lock.h @@ -25,11 +25,9 @@ struct aws_rw_lock { }; #ifdef _WIN32 -# define AWS_RW_LOCK_INIT \ - { .lock_handle = NULL } +# define AWS_RW_LOCK_INIT {.lock_handle = NULL} #else -# define AWS_RW_LOCK_INIT \ - { .lock_handle = PTHREAD_RWLOCK_INITIALIZER } +# define AWS_RW_LOCK_INIT {.lock_handle = PTHREAD_RWLOCK_INITIALIZER} #endif AWS_EXTERN_C_BEGIN diff --git a/include/aws/common/statistics.h b/include/aws/common/statistics.h index f9d432d33..efeed7228 100644 --- a/include/aws/common/statistics.h +++ b/include/aws/common/statistics.h @@ -23,7 +23,7 @@ enum { }; #define AWS_CRT_STATISTICS_CATEGORY_STRIDE (1U << AWS_CRT_STATISTICS_CATEGORY_STRIDE_BITS) -#define AWS_CRT_STATISTICS_CATEGORY_BEGIN_RANGE(x) ((x)*AWS_CRT_STATISTICS_CATEGORY_STRIDE) +#define AWS_CRT_STATISTICS_CATEGORY_BEGIN_RANGE(x) ((x) * AWS_CRT_STATISTICS_CATEGORY_STRIDE) #define AWS_CRT_STATISTICS_CATEGORY_END_RANGE(x) (((x) + 1) * AWS_CRT_STATISTICS_CATEGORY_STRIDE - 1) /** diff --git a/include/aws/common/thread.h b/include/aws/common/thread.h index 90f0bc363..84180e690 100644 --- a/include/aws/common/thread.h +++ b/include/aws/common/thread.h @@ -90,8 +90,7 @@ struct aws_thread_options { typedef union { void *ptr; } aws_thread_once; -# define AWS_THREAD_ONCE_STATIC_INIT \ - { NULL } +# define AWS_THREAD_ONCE_STATIC_INIT {NULL} typedef unsigned long aws_thread_id_t; #else typedef pthread_once_t aws_thread_once; diff --git a/source/allocator.c b/source/allocator.c index d3d1e98bd..71f7a64b0 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -207,7 +207,7 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { return mem; } -#define AWS_ALIGN_ROUND_UP(value, alignment) (((value) + ((alignment)-1)) & ~((alignment)-1)) +#define AWS_ALIGN_ROUND_UP(value, alignment) (((value) + ((alignment) - 1)) & ~((alignment) - 1)) void *aws_mem_acquire_many(struct aws_allocator *allocator, size_t count, ...) { diff --git a/source/android/logging.c b/source/android/logging.c index 6c2d9de53..303bf36bf 100644 --- a/source/android/logging.c +++ b/source/android/logging.c @@ -107,7 +107,9 @@ static int s_logcat_format(struct logcat_format_data *formatting_data, va_list a return AWS_OP_SUCCESS; } -static struct aws_logger_logcat { enum aws_log_level level; } s_logcat_impl; +static struct aws_logger_logcat { + enum aws_log_level level; +} s_logcat_impl; static int s_logcat_log( struct aws_logger *logger, diff --git a/source/common.c b/source/common.c index 2c971b5b4..3bedda749 100644 --- a/source/common.c +++ b/source/common.c @@ -80,7 +80,7 @@ void aws_secure_zero(void *pBuf, size_t bufsize) { #endif // #else not windows } -#define AWS_DEFINE_ERROR_INFO_COMMON(C, ES) [(C)-0x0000] = AWS_DEFINE_ERROR_INFO(C, ES, "aws-c-common") +#define AWS_DEFINE_ERROR_INFO_COMMON(C, ES) [(C) - 0x0000] = AWS_DEFINE_ERROR_INFO(C, ES, "aws-c-common") /* clang-format off */ static struct aws_error_info errors[] = { AWS_DEFINE_ERROR_INFO_COMMON( diff --git a/source/posix/clock.c b/source/posix/clock.c index b2c3bc3f0..076f4d835 100644 --- a/source/posix/clock.c +++ b/source/posix/clock.c @@ -47,7 +47,7 @@ static int (*s_gettime_fn)(clockid_t clock_id, struct timespec *tp) = NULL; static void s_do_osx_loads(void *user_data) { (void)user_data; - s_gettime_fn = (int (*)(clockid_t clock_id, struct timespec * tp)) dlsym(RTLD_DEFAULT, "clock_gettime"); + s_gettime_fn = (int (*)(clockid_t clock_id, struct timespec *tp))dlsym(RTLD_DEFAULT, "clock_gettime"); } int aws_high_res_clock_get_ticks(uint64_t *timestamp) { diff --git a/source/priority_queue.c b/source/priority_queue.c index fcea718bf..2157ffd0a 100644 --- a/source/priority_queue.c +++ b/source/priority_queue.c @@ -7,7 +7,7 @@ #include -#define PARENT_OF(index) (((index)&1) ? (index) >> 1 : (index) > 1 ? ((index)-2) >> 1 : 0) +#define PARENT_OF(index) (((index) & 1) ? (index) >> 1 : (index) > 1 ? ((index) - 2) >> 1 : 0) #define LEFT_OF(index) (((index) << 1) + 1) #define RIGHT_OF(index) (((index) << 1) + 2) diff --git a/tests/assert_test.c b/tests/assert_test.c index 52a4794f5..430a3f9e4 100644 --- a/tests/assert_test.c +++ b/tests/assert_test.c @@ -65,6 +65,7 @@ int side_effect(void) { } /* NOLINTNEXTLINE(readability-function-size) */ +/* clang-format off */ int test_asserts(int *index) { TEST_SUCCESS(null_test) {} TEST_FAILURE(null_failure_test) { @@ -160,6 +161,7 @@ int test_asserts(int *index) { return NO_MORE_TESTS; } +/* clang-format on */ void reset(void) { g_cur_testname = "UNKNOWN"; diff --git a/tests/condition_variable_test.c b/tests/condition_variable_test.c index e7ddd5f25..a59e7c2ba 100644 --- a/tests/condition_variable_test.c +++ b/tests/condition_variable_test.c @@ -65,13 +65,15 @@ static int s_test_conditional_notify_one_fn(struct aws_allocator *allocator, voi struct condition_predicate_args predicate_args = {.call_count = 0}; - struct conditional_test_data test_data = {.condition_variable_1 = AWS_CONDITION_VARIABLE_INIT, - .condition_variable_2 = AWS_CONDITION_VARIABLE_INIT, - .mutex = AWS_MUTEX_INIT, - .predicate_args = &predicate_args, - .thread_1 = 0, - .thread_2 = 0, - .thread_3 = 0}; + struct conditional_test_data test_data = { + .condition_variable_1 = AWS_CONDITION_VARIABLE_INIT, + .condition_variable_2 = AWS_CONDITION_VARIABLE_INIT, + .mutex = AWS_MUTEX_INIT, + .predicate_args = &predicate_args, + .thread_1 = 0, + .thread_2 = 0, + .thread_3 = 0, + }; ASSERT_SUCCESS(aws_mutex_lock(&test_data.mutex)); @@ -103,13 +105,15 @@ static int s_test_conditional_notify_all_fn(struct aws_allocator *allocator, voi struct condition_predicate_args predicate_args = {.call_count = 0}; - struct conditional_test_data test_data = {.condition_variable_1 = AWS_CONDITION_VARIABLE_INIT, - .condition_variable_2 = AWS_CONDITION_VARIABLE_INIT, - .mutex = AWS_MUTEX_INIT, - .predicate_args = &predicate_args, - .thread_1 = 0, - .thread_2 = 0, - .thread_3 = 0}; + struct conditional_test_data test_data = { + .condition_variable_1 = AWS_CONDITION_VARIABLE_INIT, + .condition_variable_2 = AWS_CONDITION_VARIABLE_INIT, + .mutex = AWS_MUTEX_INIT, + .predicate_args = &predicate_args, + .thread_1 = 0, + .thread_2 = 0, + .thread_3 = 0, + }; ASSERT_SUCCESS(aws_mutex_lock(&test_data.mutex)); diff --git a/tests/error_test.c b/tests/error_test.c index b7a241ec8..409dc5a34 100644 --- a/tests/error_test.c +++ b/tests/error_test.c @@ -399,13 +399,15 @@ static void s_error_thread_fn(void *arg) { static int s_error_code_cross_thread_test_fn(struct aws_allocator *allocator, void *ctx) { (void)ctx; - struct error_thread_test_data test_data = {.thread_1_code = 0, - .thread_1_get_last_code = 0, - .thread_1_encountered_count = 0, - .thread_2_code = 0, - .thread_2_get_last_code = 0, - .thread_2_encountered_count = 0, - .thread_2_id = 0}; + struct error_thread_test_data test_data = { + .thread_1_code = 0, + .thread_1_get_last_code = 0, + .thread_1_encountered_count = 0, + .thread_2_code = 0, + .thread_2_get_last_code = 0, + .thread_2_encountered_count = 0, + .thread_2_id = 0, + }; test_data.thread_1_id = aws_thread_current_thread_id(); From a8f6315499e7074d697dc95ec2f8663b95bcc048 Mon Sep 17 00:00:00 2001 From: Michael Tautschnig Date: Wed, 19 Jun 2024 19:54:00 +0200 Subject: [PATCH 30/42] Use CBMC version 5.95.1 (#1124) The upcoming CBMC version 6 release includes changes that may affect existing proofs. This PR will make sure that aws-c-common PRs are not negatively impacted by this release. After releasing CBMC version 6 we will issue a follow-up PR that will return aws-c-common to using CBMC's latest release, and will include any changes to proofs that may be necessary to support the new version. --- .github/workflows/proof_ci_resources/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/proof_ci_resources/config.yaml b/.github/workflows/proof_ci_resources/config.yaml index d2273314b..e4593e5bb 100644 --- a/.github/workflows/proof_ci_resources/config.yaml +++ b/.github/workflows/proof_ci_resources/config.yaml @@ -1,5 +1,5 @@ cadical-tag: latest -cbmc-version: latest +cbmc-version: "5.95.1" cbmc-viewer-version: latest kissat-tag: latest litani-version: latest From 7f3b33ea6915e597720c27c18ce52e9810a08177 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Wed, 19 Jun 2024 11:22:04 -0700 Subject: [PATCH 31/42] latest_submodules.py uses AWS-LC-FIPS releases in aws-crt-java (#1125) --- scripts/latest_submodules.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/latest_submodules.py b/scripts/latest_submodules.py index 47bfeaf90..a602b5a24 100755 --- a/scripts/latest_submodules.py +++ b/scripts/latest_submodules.py @@ -53,7 +53,7 @@ def get_submodules(): return sorted(submodules, key=lambda x: x['name']) -def get_release_tags(): +def get_release_tags(prefix='v'): """ Return list of release tags for current repo, sorted high to low. @@ -69,7 +69,7 @@ def get_release_tags(): for line in git_output.splitlines(): # line looks like: "e18f041a0c8d17189f2eae2a32f16e0a7a3f0f1c refs/tags/v0.5.18" match = re.match( - r'([a-f0-9]+)\s+refs/tags/(v([0-9]+)\.([0-9]+)\.([0-9]+))$', line) + r'([a-f0-9]+)\s+refs/tags/(' + prefix + r'([0-9]+)\.([0-9]+)\.([0-9]+))$', line) if not match: # skip malformed release tags continue @@ -118,7 +118,12 @@ def main(): os.chdir(os.path.join(root_path, submodule['path'])) - tags = get_release_tags() + version_prefix = 'v' + # aws-crt-java uses FIPS releases of aws-lc + if name == 'aws-lc' and 'aws-crt-java' in root_path: + version_prefix = 'AWS-LC-FIPS-' + + tags = get_release_tags(version_prefix) current_commit = get_current_commit() current_tag = get_tag_for_commit(tags, current_commit) sync_from = current_tag['version'] if current_tag else current_commit From 957c76062ab42a8d6006a0c17d07fe5aff4428df Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Wed, 19 Jun 2024 15:15:27 -0700 Subject: [PATCH 32/42] Use CBMC 6.0.0 (#1128) **Issue:** The proofs broke unexpectedly when CBMC 6.0.0 was released, since we were using "latest". We pinned the version back to 5.95.1 in a [recent PR](https://github.com/awslabs/aws-c-common/pull/1124) to get CI working again. **Description of changes:** - Upgrade to CBMC 6.0.0 - For all tools used in proofs, stop using "latest". Pin the exact version, so we're not caught off-guard by unexpected upgrades again in the future - Fix proof code to work with CBMC 6 --- .github/workflows/proof_ci_resources/config.yaml | 11 ++++++----- verification/cbmc/proofs/Makefile.common | 1 - verification/cbmc/sources/utils.c | 5 ++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/proof_ci_resources/config.yaml b/.github/workflows/proof_ci_resources/config.yaml index e4593e5bb..4a362f2db 100644 --- a/.github/workflows/proof_ci_resources/config.yaml +++ b/.github/workflows/proof_ci_resources/config.yaml @@ -1,7 +1,8 @@ -cadical-tag: latest -cbmc-version: "5.95.1" -cbmc-viewer-version: latest -kissat-tag: latest -litani-version: latest +# Use exact versions (instead of "latest") so we're not broken by surprise upgrades. +cadical-tag: "rel-2.0.0" # tag of latest release: https://github.com/arminbiere/cadical/releases +cbmc-version: "6.0.0" # semver of latest release: https://github.com/diffblue/cbmc/releases +cbmc-viewer-version: "3.8" # semver of latest release: https://github.com/model-checking/cbmc-viewer/releases +kissat-tag: "rel-3.1.1" # tag of latest release: https://github.com/arminbiere/kissat/releases +litani-version: "1.29.0" # semver of latest release: https://github.com/awslabs/aws-build-accumulator/releases proofs-dir: verification/cbmc/proofs run-cbmc-proofs-command: ./run-cbmc-proofs.py diff --git a/verification/cbmc/proofs/Makefile.common b/verification/cbmc/proofs/Makefile.common index 1c05c2ef4..e0897c04d 100644 --- a/verification/cbmc/proofs/Makefile.common +++ b/verification/cbmc/proofs/Makefile.common @@ -465,7 +465,6 @@ COMMA :=, # Set C compiler defines CBMCFLAGS += --object-bits $(CBMC_OBJECT_BITS) -COMPILE_FLAGS += --object-bits $(CBMC_OBJECT_BITS) DEFINES += -DCBMC=1 DEFINES += -DCBMC_OBJECT_BITS=$(CBMC_OBJECT_BITS) diff --git a/verification/cbmc/sources/utils.c b/verification/cbmc/sources/utils.c index 231ddc8f0..0c511ef91 100644 --- a/verification/cbmc/sources/utils.c +++ b/verification/cbmc/sources/utils.c @@ -170,4 +170,7 @@ uint64_t uninterpreted_hasher(const void *a) { return __CPROVER_uninterpreted_hasher(a); } -bool uninterpreted_predicate_fn(uint8_t value); +bool __CPROVER_uninterpreted_predicate_uint8_t(uint8_t); +bool uninterpreted_predicate_fn(uint8_t value) { + return __CPROVER_uninterpreted_predicate_uint8_t(value); +} From 0cb24e1c7e02f56cab43ffb6636cd58fd771fc55 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 20 Jun 2024 07:30:20 -0700 Subject: [PATCH 33/42] Fix default thread options for windows to not pin to any cpu_id (#1126) --- source/windows/thread.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/windows/thread.c b/source/windows/thread.c index b172178ce..6606c05be 100644 --- a/source/windows/thread.c +++ b/source/windows/thread.c @@ -21,6 +21,7 @@ static struct aws_thread_options s_default_options = { /* zero will make sure whatever the default for that version of windows is used. */ .stack_size = 0, + .cpu_id = -1, .join_strategy = AWS_TJS_MANUAL, }; From 6d974f92c1d86391c1dcb1173239adf757c52b2d Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Thu, 20 Jun 2024 11:25:49 -0700 Subject: [PATCH 34/42] cbor support (#1131) --- .clang-tidy | 2 +- CMakeLists.txt | 11 +- THIRD-PARTY-LICENSES.txt | 26 + include/aws/common/cbor.h | 449 ++++++++++++ include/aws/common/error.h | 2 + include/aws/common/logging.h | 1 + include/aws/common/macros.h | 18 + include/aws/common/private/byte_buf.h | 19 + .../{json_impl.h => external_module_impl.h} | 10 +- scripts/import_libcbor.py | 129 ++++ source/byte_buf.c | 19 + source/cbor.c | 647 ++++++++++++++++++ source/common.c | 12 +- source/external/libcbor/allocators.c | 19 + source/external/libcbor/cbor.c | 425 ++++++++++++ source/external/libcbor/cbor.h | 74 ++ source/external/libcbor/cbor/arrays.c | 131 ++++ source/external/libcbor/cbor/arrays.h | 137 ++++ source/external/libcbor/cbor/bytestrings.c | 119 ++++ source/external/libcbor/cbor/bytestrings.h | 150 ++++ source/external/libcbor/cbor/callbacks.c | 121 ++++ source/external/libcbor/cbor/callbacks.h | 189 +++++ source/external/libcbor/cbor/cbor_export.h | 14 + source/external/libcbor/cbor/common.c | 163 +++++ source/external/libcbor/cbor/common.h | 339 +++++++++ source/external/libcbor/cbor/configuration.h | 46 ++ source/external/libcbor/cbor/data.h | 264 +++++++ source/external/libcbor/cbor/encoding.c | 200 ++++++ source/external/libcbor/cbor/encoding.h | 140 ++++ source/external/libcbor/cbor/floats_ctrls.c | 189 +++++ source/external/libcbor/cbor/floats_ctrls.h | 240 +++++++ .../libcbor/cbor/internal/builder_callbacks.c | 422 ++++++++++++ .../libcbor/cbor/internal/builder_callbacks.h | 85 +++ .../external/libcbor/cbor/internal/encoders.c | 98 +++ .../external/libcbor/cbor/internal/encoders.h | 41 ++ .../external/libcbor/cbor/internal/loaders.c | 80 +++ .../external/libcbor/cbor/internal/loaders.h | 43 ++ .../libcbor/cbor/internal/memory_utils.c | 57 ++ .../libcbor/cbor/internal/memory_utils.h | 50 ++ source/external/libcbor/cbor/internal/stack.c | 33 + source/external/libcbor/cbor/internal/stack.h | 53 ++ .../external/libcbor/cbor/internal/unicode.c | 95 +++ .../external/libcbor/cbor/internal/unicode.h | 33 + source/external/libcbor/cbor/ints.c | 190 +++++ source/external/libcbor/cbor/ints.h | 211 ++++++ source/external/libcbor/cbor/maps.c | 125 ++++ source/external/libcbor/cbor/maps.h | 121 ++++ source/external/libcbor/cbor/serialization.c | 368 ++++++++++ source/external/libcbor/cbor/serialization.h | 168 +++++ source/external/libcbor/cbor/streaming.c | 600 ++++++++++++++++ source/external/libcbor/cbor/streaming.h | 37 + source/external/libcbor/cbor/strings.c | 142 ++++ source/external/libcbor/cbor/strings.h | 183 +++++ source/external/libcbor/cbor/tags.c | 46 ++ source/external/libcbor/cbor/tags.h | 74 ++ source/json.c | 2 +- tests/CMakeLists.txt | 11 + tests/byte_buf_test.c | 59 +- tests/cbor_test.c | 489 +++++++++++++ tests/fuzz/cbor_decoding_transitive.c | 87 +++ tests/fuzz/cbor_double_encode_decode.c | 66 ++ 61 files changed, 8366 insertions(+), 8 deletions(-) create mode 100644 include/aws/common/cbor.h rename include/aws/common/private/{json_impl.h => external_module_impl.h} (64%) create mode 100644 scripts/import_libcbor.py create mode 100644 source/cbor.c create mode 100644 source/external/libcbor/allocators.c create mode 100644 source/external/libcbor/cbor.c create mode 100644 source/external/libcbor/cbor.h create mode 100644 source/external/libcbor/cbor/arrays.c create mode 100644 source/external/libcbor/cbor/arrays.h create mode 100644 source/external/libcbor/cbor/bytestrings.c create mode 100644 source/external/libcbor/cbor/bytestrings.h create mode 100644 source/external/libcbor/cbor/callbacks.c create mode 100644 source/external/libcbor/cbor/callbacks.h create mode 100644 source/external/libcbor/cbor/cbor_export.h create mode 100644 source/external/libcbor/cbor/common.c create mode 100644 source/external/libcbor/cbor/common.h create mode 100644 source/external/libcbor/cbor/configuration.h create mode 100644 source/external/libcbor/cbor/data.h create mode 100644 source/external/libcbor/cbor/encoding.c create mode 100644 source/external/libcbor/cbor/encoding.h create mode 100644 source/external/libcbor/cbor/floats_ctrls.c create mode 100644 source/external/libcbor/cbor/floats_ctrls.h create mode 100644 source/external/libcbor/cbor/internal/builder_callbacks.c create mode 100644 source/external/libcbor/cbor/internal/builder_callbacks.h create mode 100644 source/external/libcbor/cbor/internal/encoders.c create mode 100644 source/external/libcbor/cbor/internal/encoders.h create mode 100644 source/external/libcbor/cbor/internal/loaders.c create mode 100644 source/external/libcbor/cbor/internal/loaders.h create mode 100644 source/external/libcbor/cbor/internal/memory_utils.c create mode 100644 source/external/libcbor/cbor/internal/memory_utils.h create mode 100644 source/external/libcbor/cbor/internal/stack.c create mode 100644 source/external/libcbor/cbor/internal/stack.h create mode 100644 source/external/libcbor/cbor/internal/unicode.c create mode 100644 source/external/libcbor/cbor/internal/unicode.h create mode 100644 source/external/libcbor/cbor/ints.c create mode 100644 source/external/libcbor/cbor/ints.h create mode 100644 source/external/libcbor/cbor/maps.c create mode 100644 source/external/libcbor/cbor/maps.h create mode 100644 source/external/libcbor/cbor/serialization.c create mode 100644 source/external/libcbor/cbor/serialization.h create mode 100644 source/external/libcbor/cbor/streaming.c create mode 100644 source/external/libcbor/cbor/streaming.h create mode 100644 source/external/libcbor/cbor/strings.c create mode 100644 source/external/libcbor/cbor/strings.h create mode 100644 source/external/libcbor/cbor/tags.c create mode 100644 source/external/libcbor/cbor/tags.h create mode 100644 tests/cbor_test.c create mode 100644 tests/fuzz/cbor_decoding_transitive.c create mode 100644 tests/fuzz/cbor_double_encode_decode.c diff --git a/.clang-tidy b/.clang-tidy index ca1d08ad9..af7fcd282 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,7 +1,7 @@ --- Checks: 'clang-diagnostic-*,clang-analyzer-*,readability-*,modernize-*,bugprone-*,misc-*,google-runtime-int,llvm-header-guard,fuchsia-restrict-system-includes,-clang-analyzer-valist.Uninitialized,-clang-analyzer-security.insecureAPI.rand,-clang-analyzer-alpha.*,-readability-magic-numbers,-readability-non-const-parameter,-readability-avoid-const-params-in-decls,-readability-else-after-return,-readability-isolate-declaration,-readability-uppercase-literal-suffix,-bugprone-sizeof-expression,-bugprone-easily-swappable-parameters,-readability-identifier-length,-misc-no-recursion,-readability-function-cognitive-complexity,-readability-magic-numbers' WarningsAsErrors: '*' -HeaderFilterRegex: '.*\.[h|inl]$' +HeaderFilterRegex: '^(?!.*external).*\.[h|inl]$' FormatStyle: 'file' CheckOptions: - key: readability-braces-around-statements.ShortStatementLines diff --git a/CMakeLists.txt b/CMakeLists.txt index 30826bdbf..d7cedeef9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,7 +53,12 @@ file(GLOB AWS_COMMON_SRC ) file (GLOB AWS_COMMON_EXTERNAL_SRC - "source/external/*.c") + "source/external/*.c" + "source/external/libcbor/*.c" + "source/external/libcbor/cbor/*.c" + "source/external/libcbor/cbor/internal/*.c" + ) + option(AWS_NUM_CPU_CORES "Number of CPU cores of the target machine. Useful when cross-compiling." 0) @@ -213,6 +218,10 @@ endif() set_target_properties(${PROJECT_NAME} PROPERTIES VERSION 1.0.0) set_target_properties(${PROJECT_NAME} PROPERTIES SOVERSION 1) +# libcbor files do includes like: #include "cbor/cbor_export.h" +# To make these paths work, add the location we're storing them as a search path. +target_include_directories(${PROJECT_NAME} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/source/external/libcbor) target_include_directories(${PROJECT_NAME} PUBLIC $ $) diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index 7ea3c219d..4a3e34675 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -55,3 +55,29 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------ + +** libcbor; version 0.11.0 -- https://github.com/PJK/libcbor +Copyright (c) 2014-2017 Pavel Kalvoda + +MIT License + +Copyright (c) 2014-2017 Pavel Kalvoda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/include/aws/common/cbor.h b/include/aws/common/cbor.h new file mode 100644 index 000000000..1c220ecb7 --- /dev/null +++ b/include/aws/common/cbor.h @@ -0,0 +1,449 @@ +#ifndef AWS_COMMON_CBOR_H +#define AWS_COMMON_CBOR_H + +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include + +AWS_PUSH_SANE_WARNING_LEVEL +AWS_EXTERN_C_BEGIN + +/** + * The types use by APIs, not 1:1 with major type. + * It's an extension for cbor major type in RFC8949 section 3.1 + * Major type 0 - AWS_CBOR_TYPE_UINT + * Major type 1 - AWS_CBOR_TYPE_NEGINT + * Major type 2 - AWS_CBOR_TYPE_BYTES/AWS_CBOR_TYPE_INDEF_BYTES_START + * Major type 3 - AWS_CBOR_TYPE_TEXT/AWS_CBOR_TYPE_INDEF_TEXT_START + * Major type 4 - AWS_CBOR_TYPE_ARRAY_START/AWS_CBOR_TYPE_INDEF_ARRAY_START + * Major type 5 - AWS_CBOR_TYPE_MAP_START/AWS_CBOR_TYPE_INDEF_MAP_START + * Major type 6 - AWS_CBOR_TYPE_TAG + * Major type 7 + * - 20/21 - AWS_CBOR_TYPE_BOOL + * - 22 - AWS_CBOR_TYPE_NULL + * - 23 - AWS_CBOR_TYPE_UNDEFINED + * - 25/26/27 - AWS_CBOR_TYPE_FLOAT + * - 31 - AWS_CBOR_TYPE_BREAK + * - rest of value are not supported. + */ +enum aws_cbor_type { + AWS_CBOR_TYPE_UNKNOWN = 0, + + AWS_CBOR_TYPE_UINT, + AWS_CBOR_TYPE_NEGINT, + AWS_CBOR_TYPE_FLOAT, + AWS_CBOR_TYPE_BYTES, + AWS_CBOR_TYPE_TEXT, + + AWS_CBOR_TYPE_ARRAY_START, + AWS_CBOR_TYPE_MAP_START, + + AWS_CBOR_TYPE_TAG, + + AWS_CBOR_TYPE_BOOL, + AWS_CBOR_TYPE_NULL, + AWS_CBOR_TYPE_UNDEFINED, + AWS_CBOR_TYPE_BREAK, + + AWS_CBOR_TYPE_INDEF_BYTES_START, + AWS_CBOR_TYPE_INDEF_TEXT_START, + AWS_CBOR_TYPE_INDEF_ARRAY_START, + AWS_CBOR_TYPE_INDEF_MAP_START, +}; + +/** + * The common tags, refer to RFC8949 section 3.4 + * Expected value type followed by the tag: + * AWS_CBOR_TAG_STANDARD_TIME - AWS_CBOR_TYPE_TEXT + * AWS_CBOR_TAG_EPOCH_TIME - AWS_CBOR_TYPE_UINT/AWS_CBOR_TYPE_NEGINT/AWS_CBOR_TYPE_FLOAT + * AWS_CBOR_TAG_UNSIGNED_BIGNUM - AWS_CBOR_TYPE_BYTES + * AWS_CBOR_TAG_NEGATIVE_BIGNUM - AWS_CBOR_TYPE_BYTES + * AWS_CBOR_TAG_DECIMAL_FRACTION - AWS_CBOR_TYPE_ARRAY_START/AWS_CBOR_TYPE_INDEF_ARRAY_START + **/ +#define AWS_CBOR_TAG_STANDARD_TIME 0 +#define AWS_CBOR_TAG_EPOCH_TIME 1 +#define AWS_CBOR_TAG_UNSIGNED_BIGNUM 2 +#define AWS_CBOR_TAG_NEGATIVE_BIGNUM 3 +#define AWS_CBOR_TAG_DECIMAL_FRACTION 4 + +struct aws_cbor_encoder; +struct aws_cbor_decoder; + +/******************************************************************************* + * ENCODE + ******************************************************************************/ + +/* Return c-string for aws_cbor_type */ +AWS_COMMON_API +const char *aws_cbor_type_cstr(enum aws_cbor_type type); + +/** + * @brief Create a new cbor encoder. Creating a encoder with a temporay buffer. + * Every aws_cbor_encoder_write_* will encode directly into the buffer to follow the encoded data. + * + * @param allocator + * @return aws_cbor_encoder + */ +AWS_COMMON_API +struct aws_cbor_encoder *aws_cbor_encoder_new(struct aws_allocator *allocator); + +AWS_COMMON_API +struct aws_cbor_encoder *aws_cbor_encoder_destroy(struct aws_cbor_encoder *encoder); + +/** + * @brief Get the current encoded data from encoder. The encoded data has the same lifetime as the encoder, and once + * any other function call invoked for the encoder, the encoded data is no longer valid. + * + * @param encoder + * @return struct aws_byte_cursor from the encoder buffer. + */ +AWS_COMMON_API +struct aws_byte_cursor aws_cbor_encoder_get_encoded_data(const struct aws_cbor_encoder *encoder); + +/** + * @brief Clear the current encoded buffer from encoder. + * + * @param encoder + */ +AWS_COMMON_API +void aws_cbor_encoder_reset(struct aws_cbor_encoder *encoder); + +/** + * @brief Encode a AWS_CBOR_TYPE_UINT value to "smallest possible" in encoder's buffer. + * Referring to RFC8949 section 4.2.1 + * + * TODO: maybe add a width of the encoded value. + * + * @param encoder + * @param value value to encode. + */ +AWS_COMMON_API +void aws_cbor_encoder_write_uint(struct aws_cbor_encoder *encoder, uint64_t value); + +/** + * @brief Encode a AWS_CBOR_TYPE_NEGINT value to "smallest possible" in encoder's buffer. + * It represents (-1 - value). + * Referring to RFC8949 section 4.2.1 + * + * + * @param encoder + * @param value The argument to encode to negative integer, which is (-1 - expected_val) + */ +AWS_COMMON_API +void aws_cbor_encoder_write_negint(struct aws_cbor_encoder *encoder, uint64_t value); + +/** + * @brief Encode a AWS_CBOR_TYPE_FLOAT value to "smallest possible", but will not be encoded into half-precision float, + * as it's not well supported cross languages. + * + * To be more specific, it will be encoded into integer/negative/float + * (Order with priority) when the conversation will not cause precision loss. + * + * @param encoder + * @param value value to encode. + */ +AWS_COMMON_API +void aws_cbor_encoder_write_float(struct aws_cbor_encoder *encoder, double value); + +/** + * @brief Encode a AWS_CBOR_TYPE_BYTES value to "smallest possible" in encoder's buffer. + * Referring to RFC8949 section 4.2.1, the length of "from" will be encoded first and then the value of "from" will + * be followed. + * + * @param encoder + * @param from value to encode. + */ +AWS_COMMON_API +void aws_cbor_encoder_write_bytes(struct aws_cbor_encoder *encoder, struct aws_byte_cursor from); + +/** + * @brief Encode a AWS_CBOR_TYPE_TEXT value to "smallest possible" in encoder's buffer. + * Referring to RFC8949 section 4.2.1, the length of "from" will be encoded first and then the value of "from" will + * be followed. + * + * @param encoder + * @param from value to encode. + */ +AWS_COMMON_API +void aws_cbor_encoder_write_text(struct aws_cbor_encoder *encoder, struct aws_byte_cursor from); + +/** + * @brief Encode a AWS_CBOR_TYPE_ARRAY_START value to "smallest possible" in encoder's buffer. + * Referring to RFC8949 section 4.2.1 + * The "number_entries" is the cbor data items should be followed as the content of the array. + * Notes: it's user's responsibility to keep the integrity of the array to be encoded. + * + * @param encoder + * @param number_entries The number of data item in array. + */ +AWS_COMMON_API +void aws_cbor_encoder_write_array_start(struct aws_cbor_encoder *encoder, size_t number_entries); + +/** + * @brief Encode a AWS_CBOR_TYPE_MAP_START value to "smallest possible" in encoder's buffer. + * Referring to RFC8949 section 4.2.1 + * The "number_entries" is the number of pair of cbor data items as key and value should be followed as the content of + * the map. + * + * Notes: it's user's responsibility to keep the integrity of the map to be encoded. + * + * @param encoder + * @param number_entries The number of data item in map. + */ +AWS_COMMON_API +void aws_cbor_encoder_write_map_start(struct aws_cbor_encoder *encoder, size_t number_entries); + +/** + * @brief Encode a AWS_CBOR_TYPE_TAG value to "smallest possible" in encoder's buffer. + * Referring to RFC8949 section 4.2.1 + * The following cbor data item will be the content of the tagged value. + * Notes: it's user's responsibility to keep the integrity of the tagged value to follow the RFC8949 section 3.4 + * + * @param encoder + * @param tag_number The tag value to encode. + */ +AWS_COMMON_API +void aws_cbor_encoder_write_tag(struct aws_cbor_encoder *encoder, uint64_t tag_number); + +/** + * @brief Encode a simple value AWS_CBOR_TYPE_NULL + * + * @param encoder + */ +AWS_COMMON_API +void aws_cbor_encoder_write_null(struct aws_cbor_encoder *encoder); + +/** + * @brief Encode a simple value AWS_CBOR_TYPE_UNDEFINED + * + * @param encoder + */ +AWS_COMMON_API +void aws_cbor_encoder_write_undefined(struct aws_cbor_encoder *encoder); + +/** + * @brief Encode a simple value AWS_CBOR_TYPE_BOOL + * + * @param encoder + */ +AWS_COMMON_API +void aws_cbor_encoder_write_bool(struct aws_cbor_encoder *encoder, bool value); + +/** + * @brief Encode a simple value AWS_CBOR_TYPE_BREAK + * + * Notes: no error checking, it's user's responsibility to track the break + * to close the corresponding indef_start + */ +AWS_COMMON_API +void aws_cbor_encoder_write_break(struct aws_cbor_encoder *encoder); + +/** + * @brief Encode a AWS_CBOR_TYPE_INDEF_BYTES_START + * + * Notes: no error checking, it's user's responsibility to add corresponding data and the break + * to close the indef_start + */ +AWS_COMMON_API +void aws_cbor_encoder_write_indef_bytes_start(struct aws_cbor_encoder *encoder); +/** + * @brief Encode a AWS_CBOR_TYPE_INDEF_TEXT_START + * + * Notes: no error checking, it's user's responsibility to add corresponding data + * and the break to close the indef_start + */ +AWS_COMMON_API +void aws_cbor_encoder_write_indef_text_start(struct aws_cbor_encoder *encoder); +/** + * @brief Encode a AWS_CBOR_TYPE_INDEF_ARRAY_START + * + * Notes: no error checking, it's user's responsibility to add corresponding data + * and the break to close the indef_start + */ +AWS_COMMON_API +void aws_cbor_encoder_write_indef_array_start(struct aws_cbor_encoder *encoder); +/** + * @brief Encode a AWS_CBOR_TYPE_INDEF_MAP_START + * + * Notes: no error checking, it's user's responsibility to add corresponding data + * and the break to close the indef_start + */ +AWS_COMMON_API +void aws_cbor_encoder_write_indef_map_start(struct aws_cbor_encoder *encoder); + +/******************************************************************************* + * DECODE + ******************************************************************************/ + +/** + * @brief Create a cbor decoder to take src to decode. + * The typical usage of decoder will be: + * - If the next element type only accept what expected, `aws_cbor_decoder_pop_next_*` + * - If the next element type accept different type, invoke `aws_cbor_decoder_peek_type` first, then based on the type + * to invoke corresponding `aws_cbor_decoder_pop_next_*` + * - If the next element type doesn't have corrsponding value, specifically: AWS_CBOR_TYPE_NULL, + * AWS_CBOR_TYPE_UNDEFINED, AWS_CBOR_TYPE_INF_*_START, AWS_CBOR_TYPE_BREAK, call + * `aws_cbor_decoder_consume_next_single_element` to consume it and continues for further decoding. + * - To ignore the next data item (the element and the content of it), `aws_cbor_decoder_consume_next_whole_data_item` + * + * Note: it's caller's responsibilty to keep the src outlive the decoder. + * + * @param allocator + * @param src The src data to decode from. + * @return decoder + */ +AWS_COMMON_API +struct aws_cbor_decoder *aws_cbor_decoder_new(struct aws_allocator *allocator, struct aws_byte_cursor src); + +AWS_COMMON_API +struct aws_cbor_decoder *aws_cbor_decoder_destroy(struct aws_cbor_decoder *decoder); + +/** + * @brief Get the length of the remaining bytes of the source. Once the source was decoded, it will be consumed, + * and result in decrease of the remaining length of bytes. + * + * @param decoder + * @return The length of bytes remaining of the decoder source. + */ +AWS_COMMON_API +size_t aws_cbor_decoder_get_remaining_length(const struct aws_cbor_decoder *decoder); + +/** + * @brief Decode the next element and store it in the decoder cache if there was no element cached. + * If there was element cached, just return the type of the cached element. + * + * @param decoder + * @param out_type + * @return AWS_OP_SUCCESS if succeed, AWS_OP_ERR for any decoding error and corresponding error code will be raised. + */ +AWS_COMMON_API +int aws_cbor_decoder_peek_type(struct aws_cbor_decoder *decoder, enum aws_cbor_type *out_type); + +/** + * @brief Consume the next data item, includes all the content within the data item. + * + * As an example for the following cbor, this function will consume all the data + * as it's only one cbor data item, an indefinite map with 2 key, value pair: + * 0xbf6346756ef563416d7421ff + * BF -- Start indefinite-length map + * 63 -- First key, UTF-8 string length 3 + * 46756e -- "Fun" + * F5 -- First value, true + * 63 -- Second key, UTF-8 string length 3 + * 416d74 -- "Amt" + * 21 -- Second value, -2 + * FF -- "break" + * + * Notes: this function will not ensure the data item is well-formed. + * + * @param src The src to parse data from + * @return AWS_OP_SUCCESS successfully consumed the next data item, otherwise AWS_OP_ERR. + */ +AWS_COMMON_API +int aws_cbor_decoder_consume_next_whole_data_item(struct aws_cbor_decoder *decoder); + +/** + * @brief Consume the next single element, without the content followed by the element. + * + * As an example for the following cbor, this function will only consume the + * 0xBF, "Start indefinite-length map", not any content of the map represented. + * The next element to decode will start from 0x63 + * 0xbf6346756ef563416d7421ff + * BF -- Start indefinite-length map + * 63 -- First key, UTF-8 string length 3 + * 46756e -- "Fun" + * F5 -- First value, true + * 63 -- Second key, UTF-8 string length 3 + * 416d74 -- "Amt" + * 21 -- Second value, -2 + * FF -- "break" + * + * @param decoder The decoder to parse data from + * @return AWS_OP_SUCCESS successfully consumed the next element, otherwise AWS_OP_ERR. + */ +AWS_COMMON_API +int aws_cbor_decoder_consume_next_single_element(struct aws_cbor_decoder *decoder); + +/** + * @brief Get the next element based on the type. If the next element doesn't match the expected type. Error will be + * raised. If the next element already been cached, it will consume the cached item when no error was returned. + * Specifically: + * AWS_CBOR_TYPE_UINT - aws_cbor_decoder_pop_next_unsigned_int_val + * AWS_CBOR_TYPE_NEGINT - aws_cbor_decoder_pop_next_negative_int_val, it represents (-1 - *out) + * AWS_CBOR_TYPE_FLOAT - aws_cbor_decoder_pop_next_double_val + * AWS_CBOR_TYPE_BYTES - aws_cbor_decoder_pop_next_bytes_val + * AWS_CBOR_TYPE_TEXT - aws_cbor_decoder_pop_next_text_val + * + * @param decoder + * @param out + * @return AWS_OP_SUCCESS successfully consumed the next element and get the result, otherwise AWS_OP_ERR. + */ +AWS_COMMON_API +int aws_cbor_decoder_pop_next_unsigned_int_val(struct aws_cbor_decoder *decoder, uint64_t *out); +AWS_COMMON_API +int aws_cbor_decoder_pop_next_negative_int_val(struct aws_cbor_decoder *decoder, uint64_t *out); +AWS_COMMON_API +int aws_cbor_decoder_pop_next_float_val(struct aws_cbor_decoder *decoder, double *out); +AWS_COMMON_API +int aws_cbor_decoder_pop_next_boolean_val(struct aws_cbor_decoder *decoder, bool *out); +AWS_COMMON_API +int aws_cbor_decoder_pop_next_bytes_val(struct aws_cbor_decoder *decoder, struct aws_byte_cursor *out); +AWS_COMMON_API +int aws_cbor_decoder_pop_next_text_val(struct aws_cbor_decoder *decoder, struct aws_byte_cursor *out); + +/** + * @brief Get the next AWS_CBOR_TYPE_ARRAY_START element. Only consume the AWS_CBOR_TYPE_ARRAY_START element and set the + * size of array to *out_size, not the content of the array. The next *out_size cbor data items will be the content of + * the array for a valid cbor data, + * + * Notes: For indefinite-length, this function will fail with "AWS_ERROR_CBOR_UNEXPECTED_TYPE". The designed way to + * handle indefinite-length is: + * - Get AWS_CBOR_TYPE_INDEF_ARRAY_START from _peek_type + * - call `aws_cbor_decoder_consume_next_single_element` to pop the indefinite-length start. + * - Decode the next data item until AWS_CBOR_TYPE_BREAK read. + * + * @param decoder + * @param out_size store the size of array if succeed. + * @return AWS_OP_SUCCESS successfully consumed the next element and get the result, otherwise AWS_OP_ERR. + */ +AWS_COMMON_API +int aws_cbor_decoder_pop_next_array_start(struct aws_cbor_decoder *decoder, uint64_t *out_size); + +/** + * @brief Get the next AWS_CBOR_TYPE_MAP_START element. Only consume the AWS_CBOR_TYPE_MAP_START element and set the + * size of array to *out_size, not the content of the map. The next *out_size pair of cbor data items as key and value + * will be the content of the array for a valid cbor data, + * + * Notes: For indefinite-length, this function will fail with "AWS_ERROR_CBOR_UNEXPECTED_TYPE". The designed way to + * handle indefinite-length is: + * - Get AWS_CBOR_TYPE_INDEF_MAP_START from _peek_type + * - call `aws_cbor_decoder_consume_next_single_element` to pop the indefinite-length start. + * - Decode the next data item until AWS_CBOR_TYPE_BREAK read. + * + * @param decoder + * @param out_size store the size of map if succeed. + * @return AWS_OP_SUCCESS successfully consumed the next element and get the result, otherwise AWS_OP_ERR. + */ +AWS_COMMON_API +int aws_cbor_decoder_pop_next_map_start(struct aws_cbor_decoder *decoder, uint64_t *out_size); + +/** + * @brief Get the next AWS_CBOR_TYPE_TAG element. Only consume the AWS_CBOR_TYPE_TAG element and set the + * tag value to *out_tag_val, not the content of the tagged. The next cbor data item will be the content of the tagged + * value for a valid cbor data. + * + * @param decoder + * @param out_size store the size of map if succeed. + * @return AWS_OP_SUCCESS successfully consumed the next element and get the result, otherwise AWS_OP_ERR. + */ +AWS_COMMON_API +int aws_cbor_decoder_pop_next_tag_val(struct aws_cbor_decoder *decoder, uint64_t *out_tag_val); + +AWS_EXTERN_C_END +AWS_POP_SANE_WARNING_LEVEL + +#endif // AWS_COMMON_CBOR_H diff --git a/include/aws/common/error.h b/include/aws/common/error.h index 8c5d45f69..a798b6677 100644 --- a/include/aws/common/error.h +++ b/include/aws/common/error.h @@ -216,6 +216,8 @@ enum aws_common_error { AWS_ERROR_FILE_OPEN_FAILURE, AWS_ERROR_FILE_READ_FAILURE, AWS_ERROR_FILE_WRITE_FAILURE, + AWS_ERROR_INVALID_CBOR, + AWS_ERROR_CBOR_UNEXPECTED_TYPE, AWS_ERROR_END_COMMON_RANGE = AWS_ERROR_ENUM_END_RANGE(AWS_C_COMMON_PACKAGE_ID) }; diff --git a/include/aws/common/logging.h b/include/aws/common/logging.h index f71a375b5..444465aa1 100644 --- a/include/aws/common/logging.h +++ b/include/aws/common/logging.h @@ -89,6 +89,7 @@ enum aws_common_log_subject { AWS_LS_COMMON_BUS, AWS_LS_COMMON_TEST, AWS_LS_COMMON_JSON_PARSER, + AWS_LS_COMMON_CBOR, AWS_LS_COMMON_LAST = AWS_LOG_SUBJECT_END_RANGE(AWS_C_COMMON_PACKAGE_ID) }; diff --git a/include/aws/common/macros.h b/include/aws/common/macros.h index f5c64c6aa..03e50a218 100644 --- a/include/aws/common/macros.h +++ b/include/aws/common/macros.h @@ -131,6 +131,24 @@ enum { AWS_CACHE_LINE = 64 }; # define AWS_SUPPRESS_TSAN #endif +#if defined(__has_feature) +# if __has_feature(undefined_behavior_sanitizer) +# define AWS_SUPPRESS_UBSAN __attribute__((no_sanitize("undefined"))) +# endif +#elif defined(__SANITIZE_UNDEFINED__) +# if defined(__GNUC__) +# define AWS_SUPPRESS_UBSAN __attribute__((no_sanitize_undefined)) +# else +# define AWS_SUPPRESS_UBSAN +# endif +#else +# define AWS_SUPPRESS_UBSAN +#endif + +#if !defined(AWS_SUPPRESS_UBSAN) +# define AWS_SUPPRESS_UBSAN +#endif + /* If this is C++, restrict isn't supported. If this is not at least C99 on gcc and clang, it isn't supported. * If visual C++ building in C mode, the restrict definition is __restrict. * This just figures all of that out based on who's including this header file. */ diff --git a/include/aws/common/private/byte_buf.h b/include/aws/common/private/byte_buf.h index 81c08cbba..a9eebf034 100644 --- a/include/aws/common/private/byte_buf.h +++ b/include/aws/common/private/byte_buf.h @@ -15,4 +15,23 @@ */ AWS_COMMON_API size_t aws_nospec_mask(size_t index, size_t bound); +/** + * Expand the buffer appropriately to meet the requested capacity. + * + * If the the buffer's capacity is currently larger than the request capacity, the + * function does nothing (no shrink is performed). + */ +AWS_COMMON_API +int aws_byte_buf_reserve_smart(struct aws_byte_buf *buffer, size_t requested_capacity); + +/** + * Convenience function that attempts to increase the capacity of a buffer relative to the current + * length appropriately. + * + * If the the buffer's capacity is currently larger than the request capacity, the + * function does nothing (no shrink is performed). + */ +AWS_COMMON_API +int aws_byte_buf_reserve_smart_relative(struct aws_byte_buf *buffer, size_t additional_length); + #endif /* AWS_COMMON_PRIVATE_BYTE_BUF_H */ diff --git a/include/aws/common/private/json_impl.h b/include/aws/common/private/external_module_impl.h similarity index 64% rename from include/aws/common/private/json_impl.h rename to include/aws/common/private/external_module_impl.h index df9d81a5c..d3f75be39 100644 --- a/include/aws/common/private/json_impl.h +++ b/include/aws/common/private/external_module_impl.h @@ -1,5 +1,5 @@ -#ifndef AWS_COMMON_PRIVATE_JSON_IMPL_H -#define AWS_COMMON_PRIVATE_JSON_IMPL_H +#ifndef AWS_COMMON_PRIVATE_EXTERNAL_MODULE_IMPL_H +#define AWS_COMMON_PRIVATE_EXTERNAL_MODULE_IMPL_H /** * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. @@ -19,4 +19,8 @@ void aws_json_module_init(struct aws_allocator *allocator); */ void aws_json_module_cleanup(void); -#endif // AWS_COMMON_PRIVATE_JSON_IMPL_H +void aws_cbor_module_init(struct aws_allocator *allocator); + +void aws_cbor_module_cleanup(void); + +#endif // AWS_COMMON_PRIVATE_EXTERNAL_MODULE_IMPL_H diff --git a/scripts/import_libcbor.py b/scripts/import_libcbor.py new file mode 100644 index 000000000..37b5df9a3 --- /dev/null +++ b/scripts/import_libcbor.py @@ -0,0 +1,129 @@ +import os +import tempfile +import shutil +import subprocess +import argparse +import re + + +def parse_version(version_string): + match = re.fullmatch(r'v(\d+)\.(\d+)\.(\d+)', version_string) + if not match: + raise ValueError("Invalid version string") + return match.group(1), match.group(2), match.group(3) + + +argument_parser = argparse.ArgumentParser( + description="Helper to import libcbor as external dependency.") + +argument_parser.add_argument("--version", + required=True, help="Version string to import") + +args = argument_parser.parse_args() +major_version, minor_version, patch_version = parse_version(args.version) + +GENERATED_NOTES = """/** + * DO NOT DIRECTLY MODIFY THIS FILE: + * + * The code in this file is generated from scripts/import_libcbor.py + * and any modifications should be in there. + */ +""" + +CBOR_EXPORT_H = """ +#ifndef CBOR_EXPORT_H +#define CBOR_EXPORT_H + +/* Don't export anything from libcbor */ +#define CBOR_EXPORT + +#endif /* CBOR_EXPORT_H */ +""" + +CONFIGURATION_H = f""" +#ifndef LIBCBOR_CONFIGURATION_H +#define LIBCBOR_CONFIGURATION_H + +#define CBOR_MAJOR_VERSION {major_version} +#define CBOR_MINOR_VERSION {minor_version} +#define CBOR_PATCH_VERSION {patch_version} + +#define CBOR_BUFFER_GROWTH 2 +#define CBOR_MAX_STACK_SIZE 2048 +#define CBOR_PRETTY_PRINTER 1 + +#if defined(_MSC_VER) +# define CBOR_RESTRICT_SPECIFIER +#else +# define CBOR_RESTRICT_SPECIFIER restrict +#endif + +#define CBOR_INLINE_SPECIFIER + +/* Ignore the compiler warnings for libcbor. */ +#ifdef _MSC_VER +# pragma warning(disable : 4028) +# pragma warning(disable : 4715) +# pragma warning(disable : 4232) +# pragma warning(disable : 4068) +# pragma warning(disable : 4244) +# pragma warning(disable : 4701) +# pragma warning(disable : 4703) +#endif + +#ifdef __clang__ +# pragma clang diagnostic ignored "-Wreturn-type" +#elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wreturn-type" +# pragma GCC diagnostic ignored "-Wunknown-pragmas" +# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + +#endif // LIBCBOR_CONFIGURATION_H +""" + + +# Create a temporary directory for cloning the repository +temp_repo_dir = tempfile.mkdtemp() + +try: + # Clone the repository into the temporary directory + repo_url = "https://github.com/PJK/libcbor.git" + clone_command = f"git clone {repo_url} {temp_repo_dir}" + subprocess.run(clone_command, shell=True, check=True) + subprocess.run(["git", "checkout", "tags/" + args.version], + cwd=temp_repo_dir, check=True) + + # Create a separate folder for the copied files + output_dir = os.path.join( + os.path.dirname(__file__), "..", "source", "external", "libcbor") + shutil.rmtree(output_dir, ignore_errors=True) + os.makedirs(output_dir, exist_ok=True) + + # Copy files ending with .c and .h from the /src directory + src_dir = os.path.join(temp_repo_dir, "src") + for root, dirs, files in os.walk(src_dir): + dir_name = os.path.basename(root) + for file in files: + if file.endswith((".c", ".h")): + # copy the source code to ../source/external/libcbor + src_file = os.path.join(root, file) + rel_path = os.path.relpath(src_file, src_dir) + dest_file = os.path.join(output_dir, rel_path) + os.makedirs(os.path.dirname(dest_file), exist_ok=True) + shutil.copy(src_file, dest_file) + + # Use our customized configurations + with open(os.path.join(output_dir, "cbor/cbor_export.h"), "w") as file: + file.write(GENERATED_NOTES) + file.write(CBOR_EXPORT_H) + with open(os.path.join(output_dir, "cbor/configuration.h"), "w") as file: + file.write(GENERATED_NOTES) + file.write(CONFIGURATION_H) + +except Exception as e: + print(f"An error occurred: {e}") + +finally: + # Remove the temporary directory + shutil.rmtree(temp_repo_dir, ignore_errors=True) diff --git a/source/byte_buf.c b/source/byte_buf.c index b815b6bfb..da3748e81 100644 --- a/source/byte_buf.c +++ b/source/byte_buf.c @@ -816,6 +816,25 @@ int aws_byte_buf_reserve_relative(struct aws_byte_buf *buffer, size_t additional return aws_byte_buf_reserve(buffer, requested_capacity); } +int aws_byte_buf_reserve_smart(struct aws_byte_buf *buffer, size_t requested_capacity) { + + if (requested_capacity <= buffer->capacity) { + AWS_POSTCONDITION(aws_byte_buf_is_valid(buffer)); + return AWS_OP_SUCCESS; + } + size_t double_current_capacity = aws_add_size_saturating(buffer->capacity, buffer->capacity); + size_t new_capacity = aws_max_size(requested_capacity, double_current_capacity); + return aws_byte_buf_reserve(buffer, new_capacity); +} + +int aws_byte_buf_reserve_smart_relative(struct aws_byte_buf *buffer, size_t additional_length) { + size_t requested_capacity = 0; + if (AWS_UNLIKELY(aws_add_size_checked(buffer->len, additional_length, &requested_capacity))) { + return AWS_OP_ERR; + } + return aws_byte_buf_reserve_smart(buffer, requested_capacity); +} + struct aws_byte_cursor aws_byte_cursor_right_trim_pred( const struct aws_byte_cursor *source, aws_byte_predicate_fn *predicate) { diff --git a/source/cbor.c b/source/cbor.c new file mode 100644 index 000000000..b6ec4f239 --- /dev/null +++ b/source/cbor.c @@ -0,0 +1,647 @@ +#include "external/libcbor/cbor.h" +#include + +#include +#include +#include +#include +#include +#include +#include + +static bool s_aws_cbor_module_initialized = false; + +const static size_t s_cbor_element_width_64bit = 9; +const static size_t s_cbor_element_width_32bit = 5; + +enum s_cbor_simple_val { + AWS_CBOR_SIMPLE_VAL_FALSE = 20, + AWS_CBOR_SIMPLE_VAL_TRUE = 21, + AWS_CBOR_SIMPLE_VAL_NULL = 22, + AWS_CBOR_SIMPLE_VAL_UNDEFINED = 23, + AWS_CBOR_SIMPLE_VAL_BREAK = 31, +}; + +void aws_cbor_module_init(struct aws_allocator *allocator) { + (void)allocator; + if (!s_aws_cbor_module_initialized) { + /* Not allow any allocation from libcbor */ + cbor_set_allocs(NULL, NULL, NULL); + s_aws_cbor_module_initialized = true; + } +} + +void aws_cbor_module_cleanup(void) { + if (s_aws_cbor_module_initialized) { + s_aws_cbor_module_initialized = false; + } +} + +/* Return c-string for aws_cbor_type */ +const char *aws_cbor_type_cstr(enum aws_cbor_type type) { + /* clang-format off */ + switch (type) { + case (AWS_CBOR_TYPE_UINT): return "AWS_CBOR_TYPE_UINT"; + case (AWS_CBOR_TYPE_NEGINT): return "AWS_CBOR_TYPE_NEGINT"; + case (AWS_CBOR_TYPE_FLOAT): return "AWS_CBOR_TYPE_FLOAT"; + case (AWS_CBOR_TYPE_BYTES): return "AWS_CBOR_TYPE_BYTES"; + case (AWS_CBOR_TYPE_TEXT): return "AWS_CBOR_TYPE_TEXT"; + case (AWS_CBOR_TYPE_ARRAY_START): return "AWS_CBOR_TYPE_ARRAY_START"; + case (AWS_CBOR_TYPE_MAP_START): return "AWS_CBOR_TYPE_MAP_START"; + case (AWS_CBOR_TYPE_TAG): return "AWS_CBOR_TYPE_TAG"; + case (AWS_CBOR_TYPE_BOOL): return "AWS_CBOR_TYPE_BOOL"; + case (AWS_CBOR_TYPE_NULL): return "AWS_CBOR_TYPE_NULL"; + case (AWS_CBOR_TYPE_UNDEFINED): return "AWS_CBOR_TYPE_UNDEFINED"; + case (AWS_CBOR_TYPE_BREAK): return "AWS_CBOR_TYPE_BREAK"; + case (AWS_CBOR_TYPE_INDEF_BYTES_START): return "AWS_CBOR_TYPE_INDEF_BYTES_START"; + case (AWS_CBOR_TYPE_INDEF_TEXT_START): return "AWS_CBOR_TYPE_INDEF_TEXT_START"; + case (AWS_CBOR_TYPE_INDEF_ARRAY_START): return "AWS_CBOR_TYPE_INDEF_ARRAY_START"; + case (AWS_CBOR_TYPE_INDEF_MAP_START): return "AWS_CBOR_TYPE_INDEF_MAP_START"; + default: return ""; + } + /* clang-format on */ +} + +/******************************************************************************* + * ENCODE + ******************************************************************************/ + +struct aws_cbor_encoder { + struct aws_allocator *allocator; + struct aws_byte_buf encoded_buf; +}; + +struct aws_cbor_encoder *aws_cbor_encoder_new(struct aws_allocator *allocator) { + struct aws_cbor_encoder *encoder = aws_mem_calloc(allocator, 1, sizeof(struct aws_cbor_encoder)); + encoder->allocator = allocator; + aws_byte_buf_init(&encoder->encoded_buf, allocator, 256); + + return encoder; +} + +struct aws_cbor_encoder *aws_cbor_encoder_destroy(struct aws_cbor_encoder *encoder) { + aws_byte_buf_clean_up(&encoder->encoded_buf); + aws_mem_release(encoder->allocator, encoder); + return NULL; +} + +struct aws_byte_cursor aws_cbor_encoder_get_encoded_data(const struct aws_cbor_encoder *encoder) { + return aws_byte_cursor_from_buf(&encoder->encoded_buf); +} + +void aws_cbor_encoder_reset(struct aws_cbor_encoder *encoder) { + aws_byte_buf_reset(&encoder->encoded_buf, false); +} + +static uint8_t *s_get_encoder_current_position(struct aws_cbor_encoder *encoder) { + return encoder->encoded_buf.buffer + encoder->encoded_buf.len; +} + +static size_t s_get_encoder_remaining_len(struct aws_cbor_encoder *encoder) { + return encoder->encoded_buf.capacity - encoder->encoded_buf.len; +} + +/** + * @brief Marcos to ensure the encoder have enough space to encode the value into the buffer using given `fn`, and then + * encode it. + */ +#define ENCODE_THROUGH_LIBCBOR(encoder, length_to_reserve, value, fn) \ + do { \ + int error = aws_byte_buf_reserve_smart_relative(&(encoder)->encoded_buf, length_to_reserve); \ + (void)error; \ + AWS_FATAL_ASSERT(error == AWS_ERROR_SUCCESS); \ + size_t encoded_len = fn(value, s_get_encoder_current_position(encoder), s_get_encoder_remaining_len(encoder)); \ + AWS_FATAL_ASSERT((encoded_len) != 0); \ + (encoder)->encoded_buf.len += (encoded_len); \ + } while (false) + +void aws_cbor_encoder_write_uint(struct aws_cbor_encoder *encoder, uint64_t value) { + ENCODE_THROUGH_LIBCBOR(encoder, s_cbor_element_width_64bit, value, cbor_encode_uint); +} + +void aws_cbor_encoder_write_negint(struct aws_cbor_encoder *encoder, uint64_t value) { + ENCODE_THROUGH_LIBCBOR(encoder, s_cbor_element_width_64bit, value, cbor_encode_negint); +} + +void aws_cbor_encoder_write_single_float(struct aws_cbor_encoder *encoder, float value) { + ENCODE_THROUGH_LIBCBOR(encoder, s_cbor_element_width_32bit, value, cbor_encode_single); +} + +void aws_cbor_encoder_write_float(struct aws_cbor_encoder *encoder, double value) { + /** + * As suggested by AWS SDK SEP, write the float value as small as possible. But, do not encode to half-float. + * Convert the float value to integer if the conversion will not cause any precision loss. + */ + if (!isfinite(value)) { + /* For special value: NAN/INFINITY, type cast to float and encode into single float. */ + aws_cbor_encoder_write_single_float(encoder, (float)value); + return; + } + /* Conversation from int to floating-type is implementation defined if loss of precision */ + if (value <= (double)INT64_MAX && value >= (double)INT64_MIN) { + /** + * A prvalue of a floating point type can be converted to a prvalue of an integer type. The conversion + * truncates; that is, the fractional part is discarded. The behavior is undefined if the truncated value cannot + * be represented in the destination type. + * Check against the INT64 range to avoid undefined behavior + * + * Comparing against INT64_MAX instead of UINT64_MAX to simplify the code, which may loss the opportunity to + * convert the UINT64 range from double to uint64_t. However, converting double to uint64_t will not benefit the + * total length encoded. + **/ + int64_t int_value = (int64_t)value; + if (value == (double)int_value) { + if (int_value < 0) { + aws_cbor_encoder_write_negint(encoder, (uint64_t)(-1 - int_value)); + } else { + aws_cbor_encoder_write_uint(encoder, (uint64_t)(int_value)); + } + return; + } + } + if (value <= FLT_MAX && value >= -FLT_MAX) { + /* Only try to convert the value within the range of float. */ + float float_value = (float)value; + double converted_value = (double)float_value; + /* Try to cast a round trip to detect any precision loss. */ + if (value == converted_value) { + aws_cbor_encoder_write_single_float(encoder, float_value); + return; + } + } + + ENCODE_THROUGH_LIBCBOR(encoder, s_cbor_element_width_64bit, value, cbor_encode_double); +} + +void aws_cbor_encoder_write_map_start(struct aws_cbor_encoder *encoder, size_t number_entries) { + ENCODE_THROUGH_LIBCBOR(encoder, s_cbor_element_width_64bit, number_entries, cbor_encode_map_start); +} + +void aws_cbor_encoder_write_tag(struct aws_cbor_encoder *encoder, uint64_t tag_number) { + ENCODE_THROUGH_LIBCBOR(encoder, s_cbor_element_width_64bit, tag_number, cbor_encode_tag); +} + +void aws_cbor_encoder_write_array_start(struct aws_cbor_encoder *encoder, size_t number_entries) { + ENCODE_THROUGH_LIBCBOR(encoder, s_cbor_element_width_64bit, number_entries, cbor_encode_array_start); +} + +void aws_cbor_encoder_write_bytes(struct aws_cbor_encoder *encoder, struct aws_byte_cursor from) { + /* Reserve the bytes for the byte string start cbor item and the actual bytes */ + /* Encode the first cbor item for byte string */ + ENCODE_THROUGH_LIBCBOR(encoder, s_cbor_element_width_64bit + from.len, from.len, cbor_encode_bytestring_start); + /* Append the actual bytes to follow the cbor item */ + aws_byte_buf_append(&encoder->encoded_buf, &from); +} + +void aws_cbor_encoder_write_text(struct aws_cbor_encoder *encoder, struct aws_byte_cursor from) { + /* Reserve the bytes for the byte string start cbor item and the actual string */ + /* Encode the first cbor item for byte string */ + ENCODE_THROUGH_LIBCBOR(encoder, s_cbor_element_width_64bit + from.len, from.len, cbor_encode_string_start); + /* Append the actual string to follow the cbor item */ + aws_byte_buf_append(&encoder->encoded_buf, &from); +} + +void aws_cbor_encoder_write_bool(struct aws_cbor_encoder *encoder, bool value) { + /* Major type 7 (simple), value 20 (false) and 21 (true) */ + uint8_t ctrl_value = value == true ? AWS_CBOR_SIMPLE_VAL_TRUE : AWS_CBOR_SIMPLE_VAL_FALSE; + ENCODE_THROUGH_LIBCBOR(encoder, 1, ctrl_value, cbor_encode_ctrl); +} + +void aws_cbor_encoder_write_null(struct aws_cbor_encoder *encoder) { + /* Major type 7 (simple), value 22 (null) */ + ENCODE_THROUGH_LIBCBOR(encoder, 1, AWS_CBOR_SIMPLE_VAL_NULL /*null*/, cbor_encode_ctrl); +} + +void aws_cbor_encoder_write_undefined(struct aws_cbor_encoder *encoder) { + /* Major type 7 (simple), value 23 (undefined) */ + ENCODE_THROUGH_LIBCBOR(encoder, 1, AWS_CBOR_SIMPLE_VAL_UNDEFINED /*undefined*/, cbor_encode_ctrl); +} + +static void s_cbor_encoder_write_type_only(struct aws_cbor_encoder *encoder, enum aws_cbor_type type) { + /* All inf start takes 1 byte only */ + aws_byte_buf_reserve_smart_relative(&encoder->encoded_buf, 1); + size_t encoded_len = 0; + switch (type) { + case AWS_CBOR_TYPE_INDEF_BYTES_START: + encoded_len = cbor_encode_indef_bytestring_start( + s_get_encoder_current_position(encoder), s_get_encoder_remaining_len(encoder)); + break; + case AWS_CBOR_TYPE_INDEF_TEXT_START: + encoded_len = cbor_encode_indef_string_start( + s_get_encoder_current_position(encoder), s_get_encoder_remaining_len(encoder)); + break; + case AWS_CBOR_TYPE_INDEF_ARRAY_START: + encoded_len = cbor_encode_indef_array_start( + s_get_encoder_current_position(encoder), s_get_encoder_remaining_len(encoder)); + break; + case AWS_CBOR_TYPE_INDEF_MAP_START: + encoded_len = cbor_encode_indef_map_start( + s_get_encoder_current_position(encoder), s_get_encoder_remaining_len(encoder)); + break; + case AWS_CBOR_TYPE_BREAK: + encoded_len = + cbor_encode_break(s_get_encoder_current_position(encoder), s_get_encoder_remaining_len(encoder)); + break; + + default: + AWS_ASSERT(false); + break; + } + AWS_ASSERT(encoded_len == 1); + encoder->encoded_buf.len += encoded_len; +} +void aws_cbor_encoder_write_indef_bytes_start(struct aws_cbor_encoder *encoder) { + s_cbor_encoder_write_type_only(encoder, AWS_CBOR_TYPE_INDEF_BYTES_START); +} + +void aws_cbor_encoder_write_indef_text_start(struct aws_cbor_encoder *encoder) { + s_cbor_encoder_write_type_only(encoder, AWS_CBOR_TYPE_INDEF_TEXT_START); +} + +void aws_cbor_encoder_write_indef_array_start(struct aws_cbor_encoder *encoder) { + s_cbor_encoder_write_type_only(encoder, AWS_CBOR_TYPE_INDEF_ARRAY_START); +} + +void aws_cbor_encoder_write_indef_map_start(struct aws_cbor_encoder *encoder) { + s_cbor_encoder_write_type_only(encoder, AWS_CBOR_TYPE_INDEF_MAP_START); +} + +void aws_cbor_encoder_write_break(struct aws_cbor_encoder *encoder) { + s_cbor_encoder_write_type_only(encoder, AWS_CBOR_TYPE_BREAK); +} + +/******************************************************************************* + * DECODE + ******************************************************************************/ + +struct aws_cbor_decoder_context { + enum aws_cbor_type type; + + /* All the values only valid when the type is set to corresponding type. */ + union { + uint64_t unsigned_int_val; + uint64_t negative_int_val; + double float_val; + uint64_t tag_val; + bool boolean_val; + struct aws_byte_cursor bytes_val; + struct aws_byte_cursor text_val; + uint64_t map_start; + uint64_t array_start; + } u; +}; + +struct aws_cbor_decoder { + struct aws_allocator *allocator; + + struct aws_byte_cursor src; + + struct aws_cbor_decoder_context cached_context; + + /* Error code during decoding. Fail the decoding process without recovering, */ + int error_code; +}; + +struct aws_cbor_decoder *aws_cbor_decoder_new(struct aws_allocator *allocator, struct aws_byte_cursor src) { + + struct aws_cbor_decoder *decoder = aws_mem_calloc(allocator, 1, sizeof(struct aws_cbor_decoder)); + decoder->allocator = allocator; + decoder->src = src; + decoder->cached_context.type = AWS_CBOR_TYPE_UNKNOWN; + return decoder; +} + +struct aws_cbor_decoder *aws_cbor_decoder_destroy(struct aws_cbor_decoder *decoder) { + aws_mem_release(decoder->allocator, decoder); + return NULL; +} + +size_t aws_cbor_decoder_get_remaining_length(const struct aws_cbor_decoder *decoder) { + return decoder->src.len; +} + +#define LIBCBOR_VALUE_CALLBACK(field, callback_type, cbor_type) \ + static void s_##field##_callback(void *ctx, callback_type val) { \ + struct aws_cbor_decoder *decoder = ctx; \ + AWS_ASSERT((decoder)->cached_context.type == AWS_CBOR_TYPE_UNKNOWN); \ + (decoder)->cached_context.u.field = val; \ + (decoder)->cached_context.type = cbor_type; \ + } + +LIBCBOR_VALUE_CALLBACK(unsigned_int_val, uint64_t, AWS_CBOR_TYPE_UINT) +LIBCBOR_VALUE_CALLBACK(negative_int_val, uint64_t, AWS_CBOR_TYPE_NEGINT) +LIBCBOR_VALUE_CALLBACK(boolean_val, bool, AWS_CBOR_TYPE_BOOL) +LIBCBOR_VALUE_CALLBACK(float_val, double, AWS_CBOR_TYPE_FLOAT) +LIBCBOR_VALUE_CALLBACK(map_start, uint64_t, AWS_CBOR_TYPE_MAP_START) +LIBCBOR_VALUE_CALLBACK(array_start, uint64_t, AWS_CBOR_TYPE_ARRAY_START) +LIBCBOR_VALUE_CALLBACK(tag_val, uint64_t, AWS_CBOR_TYPE_TAG) + +static void s_uint8_callback(void *ctx, uint8_t data) { + s_unsigned_int_val_callback(ctx, (uint64_t)data); +} + +static void s_uint16_callback(void *ctx, uint16_t data) { + s_unsigned_int_val_callback(ctx, (uint64_t)data); +} + +static void s_uint32_callback(void *ctx, uint32_t data) { + s_unsigned_int_val_callback(ctx, (uint64_t)data); +} + +static void s_negint8_callback(void *ctx, uint8_t data) { + s_negative_int_val_callback(ctx, (uint64_t)data); +} + +static void s_negint16_callback(void *ctx, uint16_t data) { + s_negative_int_val_callback(ctx, (uint64_t)data); +} + +static void s_negint32_callback(void *ctx, uint32_t data) { + s_negative_int_val_callback(ctx, (uint64_t)data); +} + +static void s_float_callback(void *ctx, float data) { + s_float_val_callback(ctx, (double)data); +} + +static void s_bytes_callback(void *ctx, const unsigned char *cbor_data, uint64_t length) { + struct aws_cbor_decoder *decoder = ctx; + AWS_ASSERT((decoder)->cached_context.type == AWS_CBOR_TYPE_UNKNOWN); + if (length > SIZE_MAX) { + AWS_LOGF_ERROR(AWS_LS_COMMON_CBOR, "Decoded a bytes with %" PRIu64 " bytes causing overflow .", length); + decoder->error_code = AWS_ERROR_OVERFLOW_DETECTED; + return; + } + decoder->cached_context.type = AWS_CBOR_TYPE_BYTES; + decoder->cached_context.u.bytes_val.ptr = (uint8_t *)cbor_data; + decoder->cached_context.u.bytes_val.len = (size_t)length; +} + +static void s_str_callback(void *ctx, const unsigned char *cbor_data, uint64_t length) { + struct aws_cbor_decoder *decoder = ctx; + AWS_ASSERT((decoder)->cached_context.type == AWS_CBOR_TYPE_UNKNOWN); + if (length > SIZE_MAX) { + AWS_LOGF_ERROR(AWS_LS_COMMON_CBOR, "Decoded a string with %" PRIu64 " bytes causing overflow .", length); + decoder->error_code = AWS_ERROR_OVERFLOW_DETECTED; + return; + } + decoder->cached_context.type = AWS_CBOR_TYPE_TEXT; + decoder->cached_context.u.text_val.ptr = (uint8_t *)cbor_data; + decoder->cached_context.u.text_val.len = (size_t)length; +} + +#define LIBCBOR_SIMPLE_CALLBACK(field, cbor_type) \ + static void s_##field##_callback(void *ctx) { \ + struct aws_cbor_decoder *decoder = ctx; \ + AWS_ASSERT((decoder)->cached_context.type == AWS_CBOR_TYPE_UNKNOWN); \ + (decoder)->cached_context.type = cbor_type; \ + } + +LIBCBOR_SIMPLE_CALLBACK(inf_bytes, AWS_CBOR_TYPE_INDEF_BYTES_START) +LIBCBOR_SIMPLE_CALLBACK(inf_str, AWS_CBOR_TYPE_INDEF_TEXT_START) +LIBCBOR_SIMPLE_CALLBACK(inf_array, AWS_CBOR_TYPE_INDEF_ARRAY_START) +LIBCBOR_SIMPLE_CALLBACK(inf_map, AWS_CBOR_TYPE_INDEF_MAP_START) + +LIBCBOR_SIMPLE_CALLBACK(inf_break, AWS_CBOR_TYPE_BREAK) +LIBCBOR_SIMPLE_CALLBACK(undefined, AWS_CBOR_TYPE_UNDEFINED) +LIBCBOR_SIMPLE_CALLBACK(null, AWS_CBOR_TYPE_NULL) + +static struct cbor_callbacks s_callbacks = { + /** Unsigned int */ + .uint64 = s_unsigned_int_val_callback, + /** Unsigned int */ + .uint32 = s_uint32_callback, + /** Unsigned int */ + .uint16 = s_uint16_callback, + /** Unsigned int */ + .uint8 = s_uint8_callback, + + /** Negative int */ + .negint64 = s_negative_int_val_callback, + /** Negative int */ + .negint32 = s_negint32_callback, + /** Negative int */ + .negint16 = s_negint16_callback, + /** Negative int */ + .negint8 = s_negint8_callback, + + /** Indefinite byte string start */ + .byte_string_start = s_inf_bytes_callback, + /** Definite byte string */ + .byte_string = s_bytes_callback, + + /** Definite string */ + .string = s_str_callback, + /** Indefinite string start */ + .string_start = s_inf_str_callback, + + /** Definite array */ + .indef_array_start = s_inf_array_callback, + /** Indefinite array */ + .array_start = s_array_start_callback, + + /** Definite map */ + .indef_map_start = s_inf_map_callback, + /** Indefinite map */ + .map_start = s_map_start_callback, + + /** Tags */ + .tag = s_tag_val_callback, + + /** Half float */ + .float2 = s_float_callback, + /** Single float */ + .float4 = s_float_callback, + /** Double float */ + .float8 = s_float_val_callback, + /** Undef */ + .undefined = s_undefined_callback, + /** Null */ + .null = s_null_callback, + /** Bool */ + .boolean = s_boolean_val_callback, + + /** Indefinite item break */ + .indef_break = s_inf_break_callback, +}; + +/** + * decode the next element to the cached_content. + */ +static int s_cbor_decode_next_element(struct aws_cbor_decoder *decoder) { + struct cbor_decoder_result result = cbor_stream_decode(decoder->src.ptr, decoder->src.len, &s_callbacks, decoder); + switch (result.status) { + case CBOR_DECODER_NEDATA: + AWS_LOGF_ERROR( + AWS_LS_COMMON_CBOR, + "The decoder doesn't have enough data to decode the next element. At least %zu bytes more needed.", + result.required); + decoder->error_code = AWS_ERROR_INVALID_CBOR; + break; + case CBOR_DECODER_ERROR: + AWS_LOGF_ERROR(AWS_LS_COMMON_CBOR, "The cbor data is malformed to decode."); + decoder->error_code = AWS_ERROR_INVALID_CBOR; + break; + default: + break; + } + + if (decoder->error_code) { + /* Error happened during decoding */ + return aws_raise_error(decoder->error_code); + } + + aws_byte_cursor_advance(&decoder->src, result.read); + + return AWS_OP_SUCCESS; +} + +#define GET_NEXT_ITEM(field, out_type, expected_cbor_type) \ + /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ + int aws_cbor_decoder_pop_next_##field(struct aws_cbor_decoder *decoder, out_type *out) { \ + if ((decoder)->error_code) { \ + /* Error happened during decoding */ \ + return aws_raise_error((decoder)->error_code); \ + } \ + if ((decoder)->cached_context.type != AWS_CBOR_TYPE_UNKNOWN) { \ + /* There was a cached context, check if the cached one meets the expected. */ \ + goto decode_done; \ + } \ + if (s_cbor_decode_next_element(decoder)) { \ + return AWS_OP_ERR; \ + } \ + decode_done: \ + if ((decoder)->cached_context.type != (expected_cbor_type)) { \ + AWS_LOGF_ERROR( \ + AWS_LS_COMMON_CBOR, \ + "The decoder got unexpected type: %d (%s), while expecting type: %d (%s).", \ + (decoder)->cached_context.type, \ + aws_cbor_type_cstr((decoder)->cached_context.type), \ + (expected_cbor_type), \ + aws_cbor_type_cstr(expected_cbor_type)); \ + return aws_raise_error(AWS_ERROR_CBOR_UNEXPECTED_TYPE); \ + } else { \ + /* Clear the cache as we give it out. */ \ + (decoder)->cached_context.type = AWS_CBOR_TYPE_UNKNOWN; \ + *out = (decoder)->cached_context.u.field; \ + } \ + return AWS_OP_SUCCESS; \ + } + +GET_NEXT_ITEM(unsigned_int_val, uint64_t, AWS_CBOR_TYPE_UINT) +GET_NEXT_ITEM(negative_int_val, uint64_t, AWS_CBOR_TYPE_NEGINT) +GET_NEXT_ITEM(float_val, double, AWS_CBOR_TYPE_FLOAT) +GET_NEXT_ITEM(boolean_val, bool, AWS_CBOR_TYPE_BOOL) +GET_NEXT_ITEM(text_val, struct aws_byte_cursor, AWS_CBOR_TYPE_TEXT) +GET_NEXT_ITEM(bytes_val, struct aws_byte_cursor, AWS_CBOR_TYPE_BYTES) +GET_NEXT_ITEM(map_start, uint64_t, AWS_CBOR_TYPE_MAP_START) +GET_NEXT_ITEM(array_start, uint64_t, AWS_CBOR_TYPE_ARRAY_START) +GET_NEXT_ITEM(tag_val, uint64_t, AWS_CBOR_TYPE_TAG) + +int aws_cbor_decoder_peek_type(struct aws_cbor_decoder *decoder, enum aws_cbor_type *out_type) { + if (decoder->error_code) { + /* Error happened during decoding */ + return aws_raise_error(decoder->error_code); + } + + if (decoder->cached_context.type != AWS_CBOR_TYPE_UNKNOWN) { + /* There was a cached context, return the type. */ + *out_type = decoder->cached_context.type; + return AWS_OP_SUCCESS; + } + + /* Decode */ + if (s_cbor_decode_next_element(decoder)) { + return AWS_OP_ERR; + } + *out_type = decoder->cached_context.type; + return AWS_OP_SUCCESS; +} + +int aws_cbor_decoder_consume_next_whole_data_item(struct aws_cbor_decoder *decoder) { + if (decoder->error_code) { + /* Error happened during decoding */ + return aws_raise_error(decoder->error_code); + } + + if (decoder->cached_context.type == AWS_CBOR_TYPE_UNKNOWN) { + /* There was no cache, decode the next item */ + if (s_cbor_decode_next_element(decoder)) { + return AWS_OP_ERR; + } + } + switch (decoder->cached_context.type) { + case AWS_CBOR_TYPE_TAG: + /* Read the next data item */ + decoder->cached_context.type = AWS_CBOR_TYPE_UNKNOWN; + if (aws_cbor_decoder_consume_next_whole_data_item(decoder)) { + return AWS_OP_ERR; + } + break; + case AWS_CBOR_TYPE_MAP_START: { + uint64_t num_map_item = decoder->cached_context.u.map_start; + /* Reset type */ + decoder->cached_context.type = AWS_CBOR_TYPE_UNKNOWN; + for (uint64_t i = 0; i < num_map_item; i++) { + /* Key */ + if (aws_cbor_decoder_consume_next_whole_data_item(decoder)) { + return AWS_OP_ERR; + } + /* Value */ + if (aws_cbor_decoder_consume_next_whole_data_item(decoder)) { + return AWS_OP_ERR; + } + } + break; + } + case AWS_CBOR_TYPE_ARRAY_START: { + uint64_t num_array_item = decoder->cached_context.u.array_start; + /* Reset type */ + decoder->cached_context.type = AWS_CBOR_TYPE_UNKNOWN; + for (uint64_t i = 0; i < num_array_item; i++) { + /* item */ + if (aws_cbor_decoder_consume_next_whole_data_item(decoder)) { + return AWS_OP_ERR; + } + } + break; + } + case AWS_CBOR_TYPE_INDEF_BYTES_START: + case AWS_CBOR_TYPE_INDEF_TEXT_START: + case AWS_CBOR_TYPE_INDEF_ARRAY_START: + case AWS_CBOR_TYPE_INDEF_MAP_START: { + enum aws_cbor_type next_type; + /* Reset the cache for the tag val */ + decoder->cached_context.type = AWS_CBOR_TYPE_UNKNOWN; + if (aws_cbor_decoder_peek_type(decoder, &next_type)) { + return AWS_OP_ERR; + } + while (next_type != AWS_CBOR_TYPE_BREAK) { + if (aws_cbor_decoder_consume_next_whole_data_item(decoder)) { + return AWS_OP_ERR; + } + if (aws_cbor_decoder_peek_type(decoder, &next_type)) { + return AWS_OP_ERR; + } + } + break; + } + + default: + break; + } + + /* Done, just reset the cache */ + decoder->cached_context.type = AWS_CBOR_TYPE_UNKNOWN; + return AWS_OP_SUCCESS; +} + +int aws_cbor_decoder_consume_next_single_element(struct aws_cbor_decoder *decoder) { + enum aws_cbor_type out_type = 0; + if (aws_cbor_decoder_peek_type(decoder, &out_type)) { + return AWS_OP_ERR; + } + /* Reset the type to clear the cache. */ + decoder->cached_context.type = AWS_CBOR_TYPE_UNKNOWN; + return AWS_OP_SUCCESS; +} diff --git a/source/common.c b/source/common.c index 3bedda749..1822f93e2 100644 --- a/source/common.c +++ b/source/common.c @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include @@ -274,6 +274,12 @@ static struct aws_error_info errors[] = { AWS_DEFINE_ERROR_INFO_COMMON( AWS_ERROR_FILE_WRITE_FAILURE, "Failed writing to file."), + AWS_DEFINE_ERROR_INFO_COMMON( + AWS_ERROR_INVALID_CBOR, + "Malformed cbor data."), + AWS_DEFINE_ERROR_INFO_COMMON( + AWS_ERROR_CBOR_UNEXPECTED_TYPE, + "Unexpected cbor type encountered."), }; /* clang-format on */ @@ -297,6 +303,8 @@ static struct aws_log_subject_info s_common_log_subject_infos[] = { DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_IO, "common-io", "Common IO utilities"), DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_BUS, "bus", "Message bus"), DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_TEST, "test", "Unit/integration testing"), + DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_JSON_PARSER, "json-parser", "Subject for json parser specific logging"), + DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_CBOR, "cbor", "Subject for CBOR encode and decode"), }; static struct aws_log_subject_info_list s_common_log_subject_list = { @@ -315,6 +323,7 @@ void aws_common_library_init(struct aws_allocator *allocator) { aws_register_log_subject_info_list(&s_common_log_subject_list); aws_thread_initialize_thread_management(); aws_json_module_init(allocator); + aws_cbor_module_init(allocator); /* NUMA is funky and we can't rely on libnuma.so being available. We also don't want to take a hard dependency on it, * try and load it if we can. */ @@ -385,6 +394,7 @@ void aws_common_library_clean_up(void) { aws_unregister_error_info(&s_list); aws_unregister_log_subject_info_list(&s_common_log_subject_list); aws_json_module_cleanup(); + aws_cbor_module_cleanup(); #ifdef AWS_OS_LINUX if (g_libnuma_handle) { dlclose(g_libnuma_handle); diff --git a/source/external/libcbor/allocators.c b/source/external/libcbor/allocators.c new file mode 100644 index 000000000..43c5440dd --- /dev/null +++ b/source/external/libcbor/allocators.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "cbor/common.h" + +CBOR_EXPORT _cbor_malloc_t _cbor_malloc = malloc; +CBOR_EXPORT _cbor_realloc_t _cbor_realloc = realloc; +CBOR_EXPORT _cbor_free_t _cbor_free = free; + +void cbor_set_allocs(_cbor_malloc_t custom_malloc, + _cbor_realloc_t custom_realloc, _cbor_free_t custom_free) { + _cbor_malloc = custom_malloc; + _cbor_realloc = custom_realloc; + _cbor_free = custom_free; +} diff --git a/source/external/libcbor/cbor.c b/source/external/libcbor/cbor.c new file mode 100644 index 000000000..a8b4bcd7a --- /dev/null +++ b/source/external/libcbor/cbor.c @@ -0,0 +1,425 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "cbor.h" +#include "cbor/internal/builder_callbacks.h" +#include "cbor/internal/loaders.h" + +#pragma clang diagnostic push +cbor_item_t *cbor_load(cbor_data source, size_t source_size, + struct cbor_load_result *result) { + /* Context stack */ + static struct cbor_callbacks callbacks = { + .uint8 = &cbor_builder_uint8_callback, + .uint16 = &cbor_builder_uint16_callback, + .uint32 = &cbor_builder_uint32_callback, + .uint64 = &cbor_builder_uint64_callback, + + .negint8 = &cbor_builder_negint8_callback, + .negint16 = &cbor_builder_negint16_callback, + .negint32 = &cbor_builder_negint32_callback, + .negint64 = &cbor_builder_negint64_callback, + + .byte_string = &cbor_builder_byte_string_callback, + .byte_string_start = &cbor_builder_byte_string_start_callback, + + .string = &cbor_builder_string_callback, + .string_start = &cbor_builder_string_start_callback, + + .array_start = &cbor_builder_array_start_callback, + .indef_array_start = &cbor_builder_indef_array_start_callback, + + .map_start = &cbor_builder_map_start_callback, + .indef_map_start = &cbor_builder_indef_map_start_callback, + + .tag = &cbor_builder_tag_callback, + + .null = &cbor_builder_null_callback, + .undefined = &cbor_builder_undefined_callback, + .boolean = &cbor_builder_boolean_callback, + .float2 = &cbor_builder_float2_callback, + .float4 = &cbor_builder_float4_callback, + .float8 = &cbor_builder_float8_callback, + .indef_break = &cbor_builder_indef_break_callback}; + + if (source_size == 0) { + result->error.code = CBOR_ERR_NODATA; + return NULL; + } + struct _cbor_stack stack = _cbor_stack_init(); + + /* Target for callbacks */ + struct _cbor_decoder_context context = (struct _cbor_decoder_context){ + .stack = &stack, .creation_failed = false, .syntax_error = false}; + struct cbor_decoder_result decode_result; + *result = + (struct cbor_load_result){.read = 0, .error = {.code = CBOR_ERR_NONE}}; + + do { + if (source_size > result->read) { /* Check for overflows */ + decode_result = + cbor_stream_decode(source + result->read, source_size - result->read, + &callbacks, &context); + } else { + result->error = (struct cbor_error){.code = CBOR_ERR_NOTENOUGHDATA, + .position = result->read}; + goto error; + } + + switch (decode_result.status) { + case CBOR_DECODER_FINISHED: + /* Everything OK */ + { + result->read += decode_result.read; + break; + } + case CBOR_DECODER_NEDATA: + /* Data length doesn't match MTB expectation */ + { + result->error.code = CBOR_ERR_NOTENOUGHDATA; + goto error; + } + case CBOR_DECODER_ERROR: + /* Reserved/malformed item */ + { + result->error.code = CBOR_ERR_MALFORMATED; + goto error; + } + } + + if (context.creation_failed) { + /* Most likely unsuccessful allocation - our callback has failed */ + result->error.code = CBOR_ERR_MEMERROR; + goto error; + } else if (context.syntax_error) { + result->error.code = CBOR_ERR_SYNTAXERROR; + goto error; + } + } while (stack.size > 0); + + return context.root; + +error: + result->error.position = result->read; + // debug_print("Failed with decoder error %d at %d\n", result->error.code, + // result->error.position); cbor_describe(stack.top->item, stdout); + /* Free the stack */ + while (stack.size > 0) { + cbor_decref(&stack.top->item); + _cbor_stack_pop(&stack); + } + return NULL; +} + +static cbor_item_t *_cbor_copy_int(cbor_item_t *item, bool negative) { + cbor_item_t *res; + switch (cbor_int_get_width(item)) { + case CBOR_INT_8: + res = cbor_build_uint8(cbor_get_uint8(item)); + break; + case CBOR_INT_16: + res = cbor_build_uint16(cbor_get_uint16(item)); + break; + case CBOR_INT_32: + res = cbor_build_uint32(cbor_get_uint32(item)); + break; + case CBOR_INT_64: + res = cbor_build_uint64(cbor_get_uint64(item)); + break; + } + + if (negative) cbor_mark_negint(res); + + return res; +} + +static cbor_item_t *_cbor_copy_float_ctrl(cbor_item_t *item) { + // cppcheck-suppress missingReturn + switch (cbor_float_get_width(item)) { + case CBOR_FLOAT_0: + return cbor_build_ctrl(cbor_ctrl_value(item)); + case CBOR_FLOAT_16: + return cbor_build_float2(cbor_float_get_float2(item)); + case CBOR_FLOAT_32: + return cbor_build_float4(cbor_float_get_float4(item)); + case CBOR_FLOAT_64: + return cbor_build_float8(cbor_float_get_float8(item)); + } +} + +cbor_item_t *cbor_copy(cbor_item_t *item) { + // cppcheck-suppress missingReturn + switch (cbor_typeof(item)) { + case CBOR_TYPE_UINT: + return _cbor_copy_int(item, false); + case CBOR_TYPE_NEGINT: + return _cbor_copy_int(item, true); + case CBOR_TYPE_BYTESTRING: + if (cbor_bytestring_is_definite(item)) { + return cbor_build_bytestring(cbor_bytestring_handle(item), + cbor_bytestring_length(item)); + } else { + cbor_item_t *res = cbor_new_indefinite_bytestring(); + if (res == NULL) { + return NULL; + } + + for (size_t i = 0; i < cbor_bytestring_chunk_count(item); i++) { + cbor_item_t *chunk_copy = + cbor_copy(cbor_bytestring_chunks_handle(item)[i]); + if (chunk_copy == NULL) { + cbor_decref(&res); + return NULL; + } + if (!cbor_bytestring_add_chunk(res, chunk_copy)) { + cbor_decref(&chunk_copy); + cbor_decref(&res); + return NULL; + } + cbor_decref(&chunk_copy); + } + return res; + } + case CBOR_TYPE_STRING: + if (cbor_string_is_definite(item)) { + return cbor_build_stringn((const char *)cbor_string_handle(item), + cbor_string_length(item)); + } else { + cbor_item_t *res = cbor_new_indefinite_string(); + if (res == NULL) { + return NULL; + } + + for (size_t i = 0; i < cbor_string_chunk_count(item); i++) { + cbor_item_t *chunk_copy = + cbor_copy(cbor_string_chunks_handle(item)[i]); + if (chunk_copy == NULL) { + cbor_decref(&res); + return NULL; + } + if (!cbor_string_add_chunk(res, chunk_copy)) { + cbor_decref(&chunk_copy); + cbor_decref(&res); + return NULL; + } + cbor_decref(&chunk_copy); + } + return res; + } + case CBOR_TYPE_ARRAY: { + cbor_item_t *res; + if (cbor_array_is_definite(item)) { + res = cbor_new_definite_array(cbor_array_size(item)); + } else { + res = cbor_new_indefinite_array(); + } + if (res == NULL) { + return NULL; + } + + for (size_t i = 0; i < cbor_array_size(item); i++) { + cbor_item_t *entry_copy = cbor_copy(cbor_move(cbor_array_get(item, i))); + if (entry_copy == NULL) { + cbor_decref(&res); + return NULL; + } + if (!cbor_array_push(res, entry_copy)) { + cbor_decref(&entry_copy); + cbor_decref(&res); + return NULL; + } + cbor_decref(&entry_copy); + } + return res; + } + case CBOR_TYPE_MAP: { + cbor_item_t *res; + if (cbor_map_is_definite(item)) { + res = cbor_new_definite_map(cbor_map_size(item)); + } else { + res = cbor_new_indefinite_map(); + } + if (res == NULL) { + return NULL; + } + + struct cbor_pair *it = cbor_map_handle(item); + for (size_t i = 0; i < cbor_map_size(item); i++) { + cbor_item_t *key_copy = cbor_copy(it[i].key); + if (key_copy == NULL) { + cbor_decref(&res); + return NULL; + } + cbor_item_t *value_copy = cbor_copy(it[i].value); + if (value_copy == NULL) { + cbor_decref(&res); + cbor_decref(&key_copy); + return NULL; + } + if (!cbor_map_add(res, (struct cbor_pair){.key = key_copy, + .value = value_copy})) { + cbor_decref(&res); + cbor_decref(&key_copy); + cbor_decref(&value_copy); + return NULL; + } + cbor_decref(&key_copy); + cbor_decref(&value_copy); + } + return res; + } + case CBOR_TYPE_TAG: { + cbor_item_t *item_copy = cbor_copy(cbor_move(cbor_tag_item(item))); + if (item_copy == NULL) { + return NULL; + } + cbor_item_t *tag = cbor_build_tag(cbor_tag_value(item), item_copy); + cbor_decref(&item_copy); + return tag; + } + case CBOR_TYPE_FLOAT_CTRL: + return _cbor_copy_float_ctrl(item); + } +} + +#if CBOR_PRETTY_PRINTER + +#include +#include +#include + +#define __STDC_FORMAT_MACROS + +static int _pow(int b, int ex) { + if (ex == 0) return 1; + int res = b; + while (--ex > 0) res *= b; + return res; +} + +static void _cbor_type_marquee(FILE *out, char *label, int indent) { + fprintf(out, "%*.*s[%s] ", indent, indent, " ", label); +} + +static void _cbor_nested_describe(cbor_item_t *item, FILE *out, int indent) { + const int indent_offset = 4; + switch (cbor_typeof(item)) { + case CBOR_TYPE_UINT: { + _cbor_type_marquee(out, "CBOR_TYPE_UINT", indent); + fprintf(out, "Width: %dB, ", _pow(2, cbor_int_get_width(item))); + fprintf(out, "Value: %" PRIu64 "\n", cbor_get_int(item)); + break; + } + case CBOR_TYPE_NEGINT: { + _cbor_type_marquee(out, "CBOR_TYPE_NEGINT", indent); + fprintf(out, "Width: %dB, ", _pow(2, cbor_int_get_width(item))); + fprintf(out, "Value: -%" PRIu64 " - 1\n", cbor_get_int(item)); + break; + } + case CBOR_TYPE_BYTESTRING: { + _cbor_type_marquee(out, "CBOR_TYPE_BYTESTRING", indent); + if (cbor_bytestring_is_indefinite(item)) { + fprintf(out, "Indefinite, Chunks: %zu, Chunk data:\n", + cbor_bytestring_chunk_count(item)); + for (size_t i = 0; i < cbor_bytestring_chunk_count(item); i++) + _cbor_nested_describe(cbor_bytestring_chunks_handle(item)[i], out, + indent + indent_offset); + } else { + const unsigned char *data = cbor_bytestring_handle(item); + fprintf(out, "Definite, Length: %zuB, Data:\n", + cbor_bytestring_length(item)); + fprintf(out, "%*s", indent + indent_offset, " "); + for (size_t i = 0; i < cbor_bytestring_length(item); i++) + fprintf(out, "%02x", (int)(data[i] & 0xff)); + fprintf(out, "\n"); + } + break; + } + case CBOR_TYPE_STRING: { + _cbor_type_marquee(out, "CBOR_TYPE_STRING", indent); + if (cbor_string_is_indefinite(item)) { + fprintf(out, "Indefinite, Chunks: %zu, Chunk data:\n", + cbor_string_chunk_count(item)); + for (size_t i = 0; i < cbor_string_chunk_count(item); i++) + _cbor_nested_describe(cbor_string_chunks_handle(item)[i], out, + indent + indent_offset); + } else { + fprintf(out, "Definite, Length: %zuB, Codepoints: %zu, Data:\n", + cbor_string_length(item), cbor_string_codepoint_count(item)); + fprintf(out, "%*s", indent + indent_offset, " "); + // Note: The string is not escaped, whitespace and control character + // will be printed in verbatim and take effect. + fwrite(cbor_string_handle(item), sizeof(unsigned char), + cbor_string_length(item), out); + fprintf(out, "\n"); + } + break; + } + case CBOR_TYPE_ARRAY: { + _cbor_type_marquee(out, "CBOR_TYPE_ARRAY", indent); + if (cbor_array_is_definite(item)) { + fprintf(out, "Definite, Size: %zu, Contents:\n", cbor_array_size(item)); + } else { + fprintf(out, "Indefinite, Size: %zu, Contents:\n", + cbor_array_size(item)); + } + + for (size_t i = 0; i < cbor_array_size(item); i++) + _cbor_nested_describe(cbor_array_handle(item)[i], out, + indent + indent_offset); + break; + } + case CBOR_TYPE_MAP: { + _cbor_type_marquee(out, "CBOR_TYPE_MAP", indent); + if (cbor_map_is_definite(item)) { + fprintf(out, "Definite, Size: %zu, Contents:\n", cbor_map_size(item)); + } else { + fprintf(out, "Indefinite, Size: %zu, Contents:\n", cbor_map_size(item)); + } + + // TODO: Label and group keys and values + for (size_t i = 0; i < cbor_map_size(item); i++) { + fprintf(out, "%*sMap entry %zu\n", indent + indent_offset, " ", i); + _cbor_nested_describe(cbor_map_handle(item)[i].key, out, + indent + 2 * indent_offset); + _cbor_nested_describe(cbor_map_handle(item)[i].value, out, + indent + 2 * indent_offset); + } + break; + } + case CBOR_TYPE_TAG: { + _cbor_type_marquee(out, "CBOR_TYPE_TAG", indent); + fprintf(out, "Value: %" PRIu64 "\n", cbor_tag_value(item)); + _cbor_nested_describe(cbor_move(cbor_tag_item(item)), out, + indent + indent_offset); + break; + } + case CBOR_TYPE_FLOAT_CTRL: { + _cbor_type_marquee(out, "CBOR_TYPE_FLOAT_CTRL", indent); + if (cbor_float_ctrl_is_ctrl(item)) { + if (cbor_is_bool(item)) + fprintf(out, "Bool: %s\n", cbor_get_bool(item) ? "true" : "false"); + else if (cbor_is_undef(item)) + fprintf(out, "Undefined\n"); + else if (cbor_is_null(item)) + fprintf(out, "Null\n"); + else + fprintf(out, "Simple value: %d\n", cbor_ctrl_value(item)); + } else { + fprintf(out, "Width: %dB, ", _pow(2, cbor_float_get_width(item))); + fprintf(out, "Value: %lf\n", cbor_float_get_float(item)); + } + break; + } + } +} + +void cbor_describe(cbor_item_t *item, FILE *out) { + _cbor_nested_describe(item, out, 0); +} + +#endif diff --git a/source/external/libcbor/cbor.h b/source/external/libcbor/cbor.h new file mode 100644 index 000000000..46ef8f267 --- /dev/null +++ b/source/external/libcbor/cbor.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_H_ +#define LIBCBOR_H_ + +#include "cbor/common.h" +#include "cbor/data.h" + +#include "cbor/arrays.h" +#include "cbor/bytestrings.h" +#include "cbor/floats_ctrls.h" +#include "cbor/ints.h" +#include "cbor/maps.h" +#include "cbor/strings.h" +#include "cbor/tags.h" + +#include "cbor/callbacks.h" +#include "cbor/cbor_export.h" +#include "cbor/encoding.h" +#include "cbor/serialization.h" +#include "cbor/streaming.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * ============================================================================ + * High level decoding + * ============================================================================ + */ + +/** Loads data item from a buffer + * + * @param source The buffer + * @param source_size + * @param[out] result Result indicator. #CBOR_ERR_NONE on success + * @return Decoded CBOR item. The item's reference count is initialized to one. + * @return `NULL` on failure. In that case, \p result contains the location and + * description of the error. + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t* cbor_load( + cbor_data source, size_t source_size, struct cbor_load_result* result); + +/** Take a deep copy of an item + * + * All items this item points to (array and map members, string chunks, tagged + * items) will be copied recursively using #cbor_copy. The new item doesn't + * alias or point to any items from the original \p item. All the reference + * counts in the new structure are set to one. + * + * @param item item to copy + * @return Reference to the new item. The item's reference count is initialized + * to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t* cbor_copy(cbor_item_t* item); + +#if CBOR_PRETTY_PRINTER +#include + +CBOR_EXPORT void cbor_describe(cbor_item_t* item, FILE* out); +#endif + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_H_ diff --git a/source/external/libcbor/cbor/arrays.c b/source/external/libcbor/cbor/arrays.c new file mode 100644 index 000000000..a23bbe3cd --- /dev/null +++ b/source/external/libcbor/cbor/arrays.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "arrays.h" +#include +#include "internal/memory_utils.h" + +size_t cbor_array_size(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_array(item)); + return item->metadata.array_metadata.end_ptr; +} + +size_t cbor_array_allocated(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_array(item)); + return item->metadata.array_metadata.allocated; +} + +cbor_item_t *cbor_array_get(const cbor_item_t *item, size_t index) { + return cbor_incref(((cbor_item_t **)item->data)[index]); +} + +bool cbor_array_set(cbor_item_t *item, size_t index, cbor_item_t *value) { + if (index == item->metadata.array_metadata.end_ptr) { + return cbor_array_push(item, value); + } else if (index < item->metadata.array_metadata.end_ptr) { + return cbor_array_replace(item, index, value); + } else { + return false; + } +} + +bool cbor_array_replace(cbor_item_t *item, size_t index, cbor_item_t *value) { + if (index >= item->metadata.array_metadata.end_ptr) return false; + /* We cannot use cbor_array_get as that would increase the refcount */ + cbor_intermediate_decref(((cbor_item_t **)item->data)[index]); + ((cbor_item_t **)item->data)[index] = cbor_incref(value); + return true; +} + +bool cbor_array_push(cbor_item_t *array, cbor_item_t *pushee) { + CBOR_ASSERT(cbor_isa_array(array)); + struct _cbor_array_metadata *metadata = + (struct _cbor_array_metadata *)&array->metadata; + cbor_item_t **data = (cbor_item_t **)array->data; + if (cbor_array_is_definite(array)) { + /* Do not reallocate definite arrays */ + if (metadata->end_ptr >= metadata->allocated) { + return false; + } + data[metadata->end_ptr++] = pushee; + } else { + /* Exponential realloc */ + if (metadata->end_ptr >= metadata->allocated) { + // Check for overflows first + if (!_cbor_safe_to_multiply(CBOR_BUFFER_GROWTH, metadata->allocated)) { + return false; + } + + size_t new_allocation = metadata->allocated == 0 + ? 1 + : CBOR_BUFFER_GROWTH * metadata->allocated; + + unsigned char *new_data = _cbor_realloc_multiple( + array->data, sizeof(cbor_item_t *), new_allocation); + if (new_data == NULL) { + return false; + } + + array->data = new_data; + metadata->allocated = new_allocation; + } + ((cbor_item_t **)array->data)[metadata->end_ptr++] = pushee; + } + cbor_incref(pushee); + return true; +} + +bool cbor_array_is_definite(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_array(item)); + return item->metadata.array_metadata.type == _CBOR_METADATA_DEFINITE; +} + +bool cbor_array_is_indefinite(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_array(item)); + return item->metadata.array_metadata.type == _CBOR_METADATA_INDEFINITE; +} + +cbor_item_t **cbor_array_handle(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_array(item)); + return (cbor_item_t **)item->data; +} + +cbor_item_t *cbor_new_definite_array(size_t size) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t)); + _CBOR_NOTNULL(item); + cbor_item_t **data = _cbor_alloc_multiple(sizeof(cbor_item_t *), size); + _CBOR_DEPENDENT_NOTNULL(item, data); + + for (size_t i = 0; i < size; i++) { + data[i] = NULL; + } + + *item = (cbor_item_t){ + .refcount = 1, + .type = CBOR_TYPE_ARRAY, + .metadata = {.array_metadata = {.type = _CBOR_METADATA_DEFINITE, + .allocated = size, + .end_ptr = 0}}, + .data = (unsigned char *)data}; + + return item; +} + +cbor_item_t *cbor_new_indefinite_array(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t)); + _CBOR_NOTNULL(item); + + *item = (cbor_item_t){ + .refcount = 1, + .type = CBOR_TYPE_ARRAY, + .metadata = {.array_metadata = {.type = _CBOR_METADATA_INDEFINITE, + .allocated = 0, + .end_ptr = 0}}, + .data = NULL /* Can be safely realloc-ed */ + }; + return item; +} diff --git a/source/external/libcbor/cbor/arrays.h b/source/external/libcbor/cbor/arrays.h new file mode 100644 index 000000000..db19e59d0 --- /dev/null +++ b/source/external/libcbor/cbor/arrays.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_ARRAYS_H +#define LIBCBOR_ARRAYS_H + +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Get the number of members + * + * @param item An array + * @return The number of members + */ +_CBOR_NODISCARD +CBOR_EXPORT size_t cbor_array_size(const cbor_item_t* item); + +/** Get the size of the allocated storage + * + * @param item An array + * @return The size of the allocated storage (number of items) + */ +_CBOR_NODISCARD +CBOR_EXPORT size_t cbor_array_allocated(const cbor_item_t* item); + +/** Get item by index + * + * @param item An array + * @param index The index (zero-based) + * @return Reference to the item, or `NULL` in case of boundary violation. + * + * Increases the reference count of the underlying item. The returned reference + * must be released using #cbor_decref. + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_item_t* cbor_array_get(const cbor_item_t* item, size_t index); + +/** Set item by index + * + * If the index is out of bounds, the array is not modified and false is + * returned. Creating arrays with holes is not possible. + * + * @param item An array + * @param value The item to assign + * @param index The index (zero-based) + * @return `true` on success, `false` on allocation failure. + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_array_set(cbor_item_t* item, size_t index, + cbor_item_t* value); + +/** Replace item at an index + * + * The reference to the item being replaced will be released using #cbor_decref. + * + * @param item An array + * @param value The item to assign. Its reference count will be increased by + * one. + * @param index The index (zero-based) + * @return true on success, false on allocation failure. + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_array_replace(cbor_item_t* item, size_t index, + cbor_item_t* value); + +/** Is the array definite? + * + * @param item An array + * @return Is the array definite? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_array_is_definite(const cbor_item_t* item); + +/** Is the array indefinite? + * + * @param item An array + * @return Is the array indefinite? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_array_is_indefinite(const cbor_item_t* item); + +/** Get the array contents + * + * The items may be reordered and modified as long as references remain + * consistent. + * + * @param item An array item + * @return An array of #cbor_item_t pointers of size #cbor_array_size. + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_item_t** cbor_array_handle(const cbor_item_t* item); + +/** Create new definite array + * + * @param size Number of slots to preallocate + * @return Reference to the new array item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_item_t* cbor_new_definite_array(size_t size); + +/** Create new indefinite array + * + * @return Reference to the new array item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_item_t* cbor_new_indefinite_array(void); + +/** Append to the end + * + * For indefinite items, storage may be reallocated. For definite items, only + * the preallocated capacity is available. + * + * @param array An array + * @param pushee The item to push. Its reference count will be increased by + * one. + * @return `true` on success, `false` on failure + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_array_push(cbor_item_t* array, cbor_item_t* pushee); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_ARRAYS_H diff --git a/source/external/libcbor/cbor/bytestrings.c b/source/external/libcbor/cbor/bytestrings.c new file mode 100644 index 000000000..528937179 --- /dev/null +++ b/source/external/libcbor/cbor/bytestrings.c @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "bytestrings.h" +#include +#include "internal/memory_utils.h" + +size_t cbor_bytestring_length(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_bytestring(item)); + return item->metadata.bytestring_metadata.length; +} + +unsigned char *cbor_bytestring_handle(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_bytestring(item)); + return item->data; +} + +bool cbor_bytestring_is_definite(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_bytestring(item)); + return item->metadata.bytestring_metadata.type == _CBOR_METADATA_DEFINITE; +} + +bool cbor_bytestring_is_indefinite(const cbor_item_t *item) { + return !cbor_bytestring_is_definite(item); +} + +cbor_item_t *cbor_new_definite_bytestring(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t)); + _CBOR_NOTNULL(item); + *item = (cbor_item_t){ + .refcount = 1, + .type = CBOR_TYPE_BYTESTRING, + .metadata = {.bytestring_metadata = {.type = _CBOR_METADATA_DEFINITE, + .length = 0}}}; + return item; +} + +cbor_item_t *cbor_new_indefinite_bytestring(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t)); + _CBOR_NOTNULL(item); + *item = (cbor_item_t){ + .refcount = 1, + .type = CBOR_TYPE_BYTESTRING, + .metadata = {.bytestring_metadata = {.type = _CBOR_METADATA_INDEFINITE, + .length = 0}}, + .data = _cbor_malloc(sizeof(struct cbor_indefinite_string_data))}; + _CBOR_DEPENDENT_NOTNULL(item, item->data); + *((struct cbor_indefinite_string_data *)item->data) = + (struct cbor_indefinite_string_data){ + .chunk_count = 0, + .chunk_capacity = 0, + .chunks = NULL, + }; + return item; +} + +cbor_item_t *cbor_build_bytestring(cbor_data handle, size_t length) { + cbor_item_t *item = cbor_new_definite_bytestring(); + _CBOR_NOTNULL(item); + void *content = _cbor_malloc(length); + _CBOR_DEPENDENT_NOTNULL(item, content); + memcpy(content, handle, length); + cbor_bytestring_set_handle(item, content, length); + return item; +} + +void cbor_bytestring_set_handle(cbor_item_t *item, + cbor_mutable_data CBOR_RESTRICT_POINTER data, + size_t length) { + CBOR_ASSERT(cbor_isa_bytestring(item)); + CBOR_ASSERT(cbor_bytestring_is_definite(item)); + item->data = data; + item->metadata.bytestring_metadata.length = length; +} + +cbor_item_t **cbor_bytestring_chunks_handle(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_bytestring(item)); + CBOR_ASSERT(cbor_bytestring_is_indefinite(item)); + return ((struct cbor_indefinite_string_data *)item->data)->chunks; +} + +size_t cbor_bytestring_chunk_count(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_bytestring(item)); + CBOR_ASSERT(cbor_bytestring_is_indefinite(item)); + return ((struct cbor_indefinite_string_data *)item->data)->chunk_count; +} + +bool cbor_bytestring_add_chunk(cbor_item_t *item, cbor_item_t *chunk) { + CBOR_ASSERT(cbor_isa_bytestring(item)); + CBOR_ASSERT(cbor_bytestring_is_indefinite(item)); + CBOR_ASSERT(cbor_isa_bytestring(chunk)); + CBOR_ASSERT(cbor_bytestring_is_definite(chunk)); + struct cbor_indefinite_string_data *data = + (struct cbor_indefinite_string_data *)item->data; + if (data->chunk_count == data->chunk_capacity) { + if (!_cbor_safe_to_multiply(CBOR_BUFFER_GROWTH, data->chunk_capacity)) { + return false; + } + + size_t new_chunk_capacity = + data->chunk_capacity == 0 ? 1 + : CBOR_BUFFER_GROWTH * (data->chunk_capacity); + + cbor_item_t **new_chunks_data = _cbor_realloc_multiple( + data->chunks, sizeof(cbor_item_t *), new_chunk_capacity); + + if (new_chunks_data == NULL) { + return false; + } + data->chunk_capacity = new_chunk_capacity; + data->chunks = new_chunks_data; + } + data->chunks[data->chunk_count++] = cbor_incref(chunk); + return true; +} diff --git a/source/external/libcbor/cbor/bytestrings.h b/source/external/libcbor/cbor/bytestrings.h new file mode 100644 index 000000000..cacd1adf9 --- /dev/null +++ b/source/external/libcbor/cbor/bytestrings.h @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_BYTESTRINGS_H +#define LIBCBOR_BYTESTRINGS_H + +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * ============================================================================ + * Byte string manipulation + * ============================================================================ + */ + +/** Returns the length of the binary data + * + * For definite byte strings only + * + * @param item a definite bytestring + * @return length of the binary data. Zero if no chunk has been attached yet + */ +_CBOR_NODISCARD +CBOR_EXPORT size_t cbor_bytestring_length(const cbor_item_t *item); + +/** Is the byte string definite? + * + * @param item a byte string + * @return Is the byte string definite? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_bytestring_is_definite(const cbor_item_t *item); + +/** Is the byte string indefinite? + * + * @param item a byte string + * @return Is the byte string indefinite? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_bytestring_is_indefinite(const cbor_item_t *item); + +/** Get the handle to the binary data + * + * Definite items only. Modifying the data is allowed. In that case, the caller + * takes responsibility for the effect on items this item might be a part of + * + * @param item A definite byte string + * @return The address of the underlying binary data + * @return `NULL` if no data have been assigned + * yet. + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_mutable_data cbor_bytestring_handle(const cbor_item_t *item); + +/** Set the handle to the binary data + * + * @param item A definite byte string + * @param data The memory block. The caller gives up the ownership of the block. + * libcbor will deallocate it when appropriate using the `free` implementation + * configured using #cbor_set_allocs + * @param length Length of the data block + */ +CBOR_EXPORT void cbor_bytestring_set_handle( + cbor_item_t *item, cbor_mutable_data CBOR_RESTRICT_POINTER data, + size_t length); + +/** Get the handle to the array of chunks + * + * Manipulations with the memory block (e.g. sorting it) are allowed, but the + * validity and the number of chunks must be retained. + * + * @param item A indefinite byte string + * @return array of #cbor_bytestring_chunk_count definite bytestrings + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_item_t **cbor_bytestring_chunks_handle( + const cbor_item_t *item); + +/** Get the number of chunks this string consist of + * + * @param item A indefinite bytestring + * @return The chunk count. 0 for freshly created items. + */ +_CBOR_NODISCARD +CBOR_EXPORT size_t cbor_bytestring_chunk_count(const cbor_item_t *item); + +/** Appends a chunk to the bytestring + * + * Indefinite byte strings only. + * + * May realloc the chunk storage. + * + * @param item An indefinite byte string + * @param chunk A definite byte string. Its reference count will be be increased + * by one. + * @return true on success, false on realloc failure. In that case, the refcount + * of `chunk` is not increased and the `item` is left intact. + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_bytestring_add_chunk(cbor_item_t *item, + cbor_item_t *chunk); + +/** Creates a new definite byte string + * + * The handle is initialized to `NULL` and length to 0 + * + * @return Reference to the new bytestring item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_item_t *cbor_new_definite_bytestring(void); + +/** Creates a new indefinite byte string + * + * The chunks array is initialized to `NULL` and chunk count to 0 + * + * @return Reference to the new bytestring item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_item_t *cbor_new_indefinite_bytestring(void); + +/** Creates a new byte string and initializes it + * + * The `handle` will be copied to a newly allocated block + * + * @param handle Block of binary data + * @param length Length of `data` + * @return Reference to the new bytestring item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_item_t *cbor_build_bytestring(cbor_data handle, size_t length); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_BYTESTRINGS_H diff --git a/source/external/libcbor/cbor/callbacks.c b/source/external/libcbor/cbor/callbacks.c new file mode 100644 index 000000000..bdf3f79ee --- /dev/null +++ b/source/external/libcbor/cbor/callbacks.c @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "callbacks.h" + +void cbor_null_uint8_callback(void *_CBOR_UNUSED(_ctx), + uint8_t _CBOR_UNUSED(_val)) {} + +void cbor_null_uint16_callback(void *_CBOR_UNUSED(_ctx), + uint16_t _CBOR_UNUSED(_val)) {} + +void cbor_null_uint32_callback(void *_CBOR_UNUSED(_ctx), + uint32_t _CBOR_UNUSED(_val)) {} + +void cbor_null_uint64_callback(void *_CBOR_UNUSED(_ctx), + uint64_t _CBOR_UNUSED(_val)) {} + +void cbor_null_negint8_callback(void *_CBOR_UNUSED(_ctx), + uint8_t _CBOR_UNUSED(_val)) {} + +void cbor_null_negint16_callback(void *_CBOR_UNUSED(_ctx), + uint16_t _CBOR_UNUSED(_val)) {} + +void cbor_null_negint32_callback(void *_CBOR_UNUSED(_ctx), + uint32_t _CBOR_UNUSED(_val)) {} + +void cbor_null_negint64_callback(void *_CBOR_UNUSED(_ctx), + uint64_t _CBOR_UNUSED(_val)) {} + +void cbor_null_string_callback(void *_CBOR_UNUSED(_ctx), + cbor_data _CBOR_UNUSED(_val), + uint64_t _CBOR_UNUSED(_val2)) {} + +void cbor_null_string_start_callback(void *_CBOR_UNUSED(_ctx)) {} + +void cbor_null_byte_string_callback(void *_CBOR_UNUSED(_ctx), + cbor_data _CBOR_UNUSED(_val), + uint64_t _CBOR_UNUSED(_val2)) {} + +void cbor_null_byte_string_start_callback(void *_CBOR_UNUSED(_ctx)) {} + +void cbor_null_array_start_callback(void *_CBOR_UNUSED(_ctx), + uint64_t _CBOR_UNUSED(_val)) {} + +void cbor_null_indef_array_start_callback(void *_CBOR_UNUSED(_ctx)) {} + +void cbor_null_map_start_callback(void *_CBOR_UNUSED(_ctx), + uint64_t _CBOR_UNUSED(_val)) {} + +void cbor_null_indef_map_start_callback(void *_CBOR_UNUSED(_ctx)) {} + +void cbor_null_tag_callback(void *_CBOR_UNUSED(_ctx), + uint64_t _CBOR_UNUSED(_val)) {} + +void cbor_null_float2_callback(void *_CBOR_UNUSED(_ctx), + float _CBOR_UNUSED(_val)) {} + +void cbor_null_float4_callback(void *_CBOR_UNUSED(_ctx), + float _CBOR_UNUSED(_val)) {} + +void cbor_null_float8_callback(void *_CBOR_UNUSED(_ctx), + double _CBOR_UNUSED(_val)) {} + +void cbor_null_null_callback(void *_CBOR_UNUSED(_ctx)) {} + +void cbor_null_undefined_callback(void *_CBOR_UNUSED(_ctx)) {} + +void cbor_null_boolean_callback(void *_CBOR_UNUSED(_ctx), + bool _CBOR_UNUSED(_val)) {} + +void cbor_null_indef_break_callback(void *_CBOR_UNUSED(_ctx)) {} + +CBOR_EXPORT const struct cbor_callbacks cbor_empty_callbacks = { + /* Type 0 - Unsigned integers */ + .uint8 = cbor_null_uint8_callback, + .uint16 = cbor_null_uint16_callback, + .uint32 = cbor_null_uint32_callback, + .uint64 = cbor_null_uint64_callback, + + /* Type 1 - Negative integers */ + .negint8 = cbor_null_negint8_callback, + .negint16 = cbor_null_negint16_callback, + .negint32 = cbor_null_negint32_callback, + .negint64 = cbor_null_negint64_callback, + + /* Type 2 - Byte strings */ + .byte_string_start = cbor_null_byte_string_start_callback, + .byte_string = cbor_null_byte_string_callback, + + /* Type 3 - Strings */ + .string_start = cbor_null_string_start_callback, + .string = cbor_null_string_callback, + + /* Type 4 - Arrays */ + .indef_array_start = cbor_null_indef_array_start_callback, + .array_start = cbor_null_array_start_callback, + + /* Type 5 - Maps */ + .indef_map_start = cbor_null_indef_map_start_callback, + .map_start = cbor_null_map_start_callback, + + /* Type 6 - Tags */ + .tag = cbor_null_tag_callback, + + /* Type 7 - Floats & misc */ + /* Type names cannot be member names */ + .float2 = cbor_null_float2_callback, + /* 2B float is not supported in standard C */ + .float4 = cbor_null_float4_callback, + .float8 = cbor_null_float8_callback, + .undefined = cbor_null_undefined_callback, + .null = cbor_null_null_callback, + .boolean = cbor_null_boolean_callback, + + /* Shared indefinites */ + .indef_break = cbor_null_indef_break_callback, +}; diff --git a/source/external/libcbor/cbor/callbacks.h b/source/external/libcbor/cbor/callbacks.h new file mode 100644 index 000000000..c7ae20568 --- /dev/null +++ b/source/external/libcbor/cbor/callbacks.h @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_CALLBACKS_H +#define LIBCBOR_CALLBACKS_H + +#include + +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Callback prototype */ +typedef void (*cbor_int8_callback)(void *, uint8_t); + +/** Callback prototype */ +typedef void (*cbor_int16_callback)(void *, uint16_t); + +/** Callback prototype */ +typedef void (*cbor_int32_callback)(void *, uint32_t); + +/** Callback prototype */ +typedef void (*cbor_int64_callback)(void *, uint64_t); + +/** Callback prototype */ +typedef void (*cbor_simple_callback)(void *); + +/** Callback prototype */ +typedef void (*cbor_string_callback)(void *, cbor_data, uint64_t); + +/** Callback prototype */ +typedef void (*cbor_collection_callback)(void *, uint64_t); + +/** Callback prototype */ +typedef void (*cbor_float_callback)(void *, float); + +/** Callback prototype */ +typedef void (*cbor_double_callback)(void *, double); + +/** Callback prototype */ +typedef void (*cbor_bool_callback)(void *, bool); + +/** Callback bundle -- passed to the decoder */ +struct cbor_callbacks { + /** Unsigned int */ + cbor_int8_callback uint8; + /** Unsigned int */ + cbor_int16_callback uint16; + /** Unsigned int */ + cbor_int32_callback uint32; + /** Unsigned int */ + cbor_int64_callback uint64; + + /** Negative int */ + cbor_int64_callback negint64; + /** Negative int */ + cbor_int32_callback negint32; + /** Negative int */ + cbor_int16_callback negint16; + /** Negative int */ + cbor_int8_callback negint8; + + /** Definite byte string */ + cbor_simple_callback byte_string_start; + /** Indefinite byte string start */ + cbor_string_callback byte_string; + + /** Definite string */ + cbor_string_callback string; + /** Indefinite string start */ + cbor_simple_callback string_start; + + /** Definite array */ + cbor_simple_callback indef_array_start; + /** Indefinite array */ + cbor_collection_callback array_start; + + /** Definite map */ + cbor_simple_callback indef_map_start; + /** Indefinite map */ + cbor_collection_callback map_start; + + /** Tags */ + cbor_int64_callback tag; + + /** Half float */ + cbor_float_callback float2; + /** Single float */ + cbor_float_callback float4; + /** Double float */ + cbor_double_callback float8; + /** Undef */ + cbor_simple_callback undefined; + /** Null */ + cbor_simple_callback null; + /** Bool */ + cbor_bool_callback boolean; + + /** Indefinite item break */ + cbor_simple_callback indef_break; +}; + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_uint8_callback(void *, uint8_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_uint16_callback(void *, uint16_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_uint32_callback(void *, uint32_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_uint64_callback(void *, uint64_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_negint8_callback(void *, uint8_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_negint16_callback(void *, uint16_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_negint32_callback(void *, uint32_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_negint64_callback(void *, uint64_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_string_callback(void *, cbor_data, uint64_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_string_start_callback(void *); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_byte_string_callback(void *, cbor_data, uint64_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_byte_string_start_callback(void *); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_array_start_callback(void *, uint64_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_indef_array_start_callback(void *); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_map_start_callback(void *, uint64_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_indef_map_start_callback(void *); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_tag_callback(void *, uint64_t); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_float2_callback(void *, float); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_float4_callback(void *, float); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_float8_callback(void *, double); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_null_callback(void *); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_undefined_callback(void *); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_boolean_callback(void *, bool); + +/** Dummy callback implementation - does nothing */ +CBOR_EXPORT void cbor_null_indef_break_callback(void *); + +/** Dummy callback bundle - does nothing */ +CBOR_EXPORT extern const struct cbor_callbacks cbor_empty_callbacks; + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_CALLBACKS_H diff --git a/source/external/libcbor/cbor/cbor_export.h b/source/external/libcbor/cbor/cbor_export.h new file mode 100644 index 000000000..b739bb32b --- /dev/null +++ b/source/external/libcbor/cbor/cbor_export.h @@ -0,0 +1,14 @@ +/** + * DO NOT DIRECTLY MODIFY THIS FILE: + * + * The code in this file is generated from scripts/import_libcbor.py + * and any modifications should be in there. + */ + +#ifndef CBOR_EXPORT_H +#define CBOR_EXPORT_H + +/* Don't export anything from libcbor */ +#define CBOR_EXPORT + +#endif /* CBOR_EXPORT_H */ diff --git a/source/external/libcbor/cbor/common.c b/source/external/libcbor/cbor/common.c new file mode 100644 index 000000000..efbd37ed7 --- /dev/null +++ b/source/external/libcbor/cbor/common.c @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "cbor/common.h" +#include "arrays.h" +#include "bytestrings.h" +#include "data.h" +#include "floats_ctrls.h" +#include "ints.h" +#include "maps.h" +#include "strings.h" +#include "tags.h" + +#ifdef DEBUG +bool _cbor_enable_assert = true; +#endif + +bool cbor_isa_uint(const cbor_item_t *item) { + return item->type == CBOR_TYPE_UINT; +} + +bool cbor_isa_negint(const cbor_item_t *item) { + return item->type == CBOR_TYPE_NEGINT; +} + +bool cbor_isa_bytestring(const cbor_item_t *item) { + return item->type == CBOR_TYPE_BYTESTRING; +} + +bool cbor_isa_string(const cbor_item_t *item) { + return item->type == CBOR_TYPE_STRING; +} + +bool cbor_isa_array(const cbor_item_t *item) { + return item->type == CBOR_TYPE_ARRAY; +} + +bool cbor_isa_map(const cbor_item_t *item) { + return item->type == CBOR_TYPE_MAP; +} + +bool cbor_isa_tag(const cbor_item_t *item) { + return item->type == CBOR_TYPE_TAG; +} + +bool cbor_isa_float_ctrl(const cbor_item_t *item) { + return item->type == CBOR_TYPE_FLOAT_CTRL; +} + +cbor_type cbor_typeof(const cbor_item_t *item) { return item->type; } + +bool cbor_is_int(const cbor_item_t *item) { + return cbor_isa_uint(item) || cbor_isa_negint(item); +} + +bool cbor_is_bool(const cbor_item_t *item) { + return cbor_isa_float_ctrl(item) && + (cbor_ctrl_value(item) == CBOR_CTRL_FALSE || + cbor_ctrl_value(item) == CBOR_CTRL_TRUE); +} + +bool cbor_is_null(const cbor_item_t *item) { + return cbor_isa_float_ctrl(item) && cbor_ctrl_value(item) == CBOR_CTRL_NULL; +} + +bool cbor_is_undef(const cbor_item_t *item) { + return cbor_isa_float_ctrl(item) && cbor_ctrl_value(item) == CBOR_CTRL_UNDEF; +} + +bool cbor_is_float(const cbor_item_t *item) { + return cbor_isa_float_ctrl(item) && !cbor_float_ctrl_is_ctrl(item); +} + +cbor_item_t *cbor_incref(cbor_item_t *item) { + item->refcount++; + return item; +} + +void cbor_decref(cbor_item_t **item_ref) { + cbor_item_t *item = *item_ref; + CBOR_ASSERT(item->refcount > 0); + if (--item->refcount == 0) { + switch (item->type) { + case CBOR_TYPE_UINT: + /* Fallthrough */ + case CBOR_TYPE_NEGINT: + /* Combined allocation, freeing the item suffices */ + { break; } + case CBOR_TYPE_BYTESTRING: { + if (cbor_bytestring_is_definite(item)) { + _cbor_free(item->data); + } else { + /* We need to decref all chunks */ + cbor_item_t **handle = cbor_bytestring_chunks_handle(item); + for (size_t i = 0; i < cbor_bytestring_chunk_count(item); i++) + cbor_decref(&handle[i]); + _cbor_free( + ((struct cbor_indefinite_string_data *)item->data)->chunks); + _cbor_free(item->data); + } + break; + } + case CBOR_TYPE_STRING: { + if (cbor_string_is_definite(item)) { + _cbor_free(item->data); + } else { + /* We need to decref all chunks */ + cbor_item_t **handle = cbor_string_chunks_handle(item); + for (size_t i = 0; i < cbor_string_chunk_count(item); i++) + cbor_decref(&handle[i]); + _cbor_free( + ((struct cbor_indefinite_string_data *)item->data)->chunks); + _cbor_free(item->data); + } + break; + } + case CBOR_TYPE_ARRAY: { + /* Get all items and decref them */ + cbor_item_t **handle = cbor_array_handle(item); + size_t size = cbor_array_size(item); + for (size_t i = 0; i < size; i++) + if (handle[i] != NULL) cbor_decref(&handle[i]); + _cbor_free(item->data); + break; + } + case CBOR_TYPE_MAP: { + struct cbor_pair *handle = cbor_map_handle(item); + for (size_t i = 0; i < item->metadata.map_metadata.end_ptr; + i++, handle++) { + cbor_decref(&handle->key); + if (handle->value != NULL) cbor_decref(&handle->value); + } + _cbor_free(item->data); + break; + } + case CBOR_TYPE_TAG: { + if (item->metadata.tag_metadata.tagged_item != NULL) + cbor_decref(&item->metadata.tag_metadata.tagged_item); + _cbor_free(item->data); + break; + } + case CBOR_TYPE_FLOAT_CTRL: { + /* Floats have combined allocation */ + break; + } + } + _cbor_free(item); + *item_ref = NULL; + } +} + +void cbor_intermediate_decref(cbor_item_t *item) { cbor_decref(&item); } + +size_t cbor_refcount(const cbor_item_t *item) { return item->refcount; } + +cbor_item_t *cbor_move(cbor_item_t *item) { + item->refcount--; + return item; +} diff --git a/source/external/libcbor/cbor/common.h b/source/external/libcbor/cbor/common.h new file mode 100644 index 000000000..1d0b426cf --- /dev/null +++ b/source/external/libcbor/cbor/common.h @@ -0,0 +1,339 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_COMMON_H +#define LIBCBOR_COMMON_H + +#include +#include +#include +#include +#include + +#include "cbor/cbor_export.h" +#include "cbor/configuration.h" +#include "data.h" + +#ifdef __cplusplus +extern "C" { + +/** + * C99 is not a subset of C++ -- 'restrict' qualifier is not a part of the + * language. This is a workaround to keep it in C headers -- compilers allow + * linking non-restrict signatures with restrict implementations. + * + * If you know a nicer way, please do let me know. + */ +#define CBOR_RESTRICT_POINTER + +#else + +// MSVC + C++ workaround +#define CBOR_RESTRICT_POINTER CBOR_RESTRICT_SPECIFIER + +#endif + +static const uint8_t cbor_major_version = CBOR_MAJOR_VERSION; +static const uint8_t cbor_minor_version = CBOR_MINOR_VERSION; +static const uint8_t cbor_patch_version = CBOR_PATCH_VERSION; + +#define CBOR_VERSION \ + _CBOR_TO_STR(CBOR_MAJOR_VERSION) \ + "." _CBOR_TO_STR(CBOR_MINOR_VERSION) "." _CBOR_TO_STR(CBOR_PATCH_VERSION) +#define CBOR_HEX_VERSION \ + ((CBOR_MAJOR_VERSION << 16) | (CBOR_MINOR_VERSION << 8) | CBOR_PATCH_VERSION) + +/* http://stackoverflow.com/questions/1644868/c-define-macro-for-debug-printing + */ +#ifdef DEBUG +#include +#define _cbor_debug_print(fmt, ...) \ + do { \ + if (DEBUG) \ + fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, __LINE__, __func__, \ + __VA_ARGS__); \ + } while (0) +extern bool _cbor_enable_assert; +// Like `assert`, but can be dynamically disabled in tests to allow testing +// invalid behaviors. +#define CBOR_ASSERT(e) assert(!_cbor_enable_assert || (e)) +#define _CBOR_TEST_DISABLE_ASSERT(block) \ + do { \ + _cbor_enable_assert = false; \ + block _cbor_enable_assert = true; \ + } while (0) +#else +#define debug_print(fmt, ...) \ + do { \ + } while (0) +#define CBOR_ASSERT(e) +#define _CBOR_TEST_DISABLE_ASSERT(block) \ + do { \ + block \ + } while (0) +#endif + +#define _CBOR_TO_STR_(x) #x +#define _CBOR_TO_STR(x) _CBOR_TO_STR_(x) /* enables proper double expansion */ + +#ifdef __GNUC__ +#define _CBOR_UNUSED(x) __attribute__((__unused__)) x +// TODO(https://github.com/PJK/libcbor/issues/247): Prefer [[nodiscard]] if +// available +#define _CBOR_NODISCARD __attribute__((warn_unused_result)) +#elif defined(_MSC_VER) +#define _CBOR_UNUSED(x) __pragma(warning(suppress : 4100 4101)) x +#define _CBOR_NODISCARD +#else +#define _CBOR_UNUSED(x) x +#define _CBOR_NODISCARD +#endif + +typedef void *(*_cbor_malloc_t)(size_t); +typedef void *(*_cbor_realloc_t)(void *, size_t); +typedef void (*_cbor_free_t)(void *); + +CBOR_EXPORT extern _cbor_malloc_t _cbor_malloc; +CBOR_EXPORT extern _cbor_realloc_t _cbor_realloc; +CBOR_EXPORT extern _cbor_free_t _cbor_free; + +// Macro to short-circuit builder functions when memory allocation fails +#define _CBOR_NOTNULL(cbor_item) \ + do { \ + if (cbor_item == NULL) { \ + return NULL; \ + } \ + } while (0) + +// Macro to short-circuit builders when memory allocation of nested data fails +#define _CBOR_DEPENDENT_NOTNULL(cbor_item, pointer) \ + do { \ + if (pointer == NULL) { \ + _cbor_free(cbor_item); \ + return NULL; \ + } \ + } while (0) + +/** Sets the memory management routines to use. + * + * By default, libcbor will use the standard library `malloc`, `realloc`, and + * `free`. + * + * \rst + * .. warning:: This function modifies the global state and should therefore be + * used accordingly. Changing the memory handlers while allocated items exist + * will result in a ``free``/``malloc`` mismatch. This function is not thread + * safe with respect to both itself and all the other *libcbor* functions that + * work with the heap. + * + * .. note:: `realloc` implementation must correctly support `NULL` reallocation + * (see e.g. http://en.cppreference.com/w/c/memory/realloc) + * \endrst + * + * @param custom_malloc malloc implementation + * @param custom_realloc realloc implementation + * @param custom_free free implementation + */ +CBOR_EXPORT void cbor_set_allocs(_cbor_malloc_t custom_malloc, + _cbor_realloc_t custom_realloc, + _cbor_free_t custom_free); + +/* + * ============================================================================ + * Type manipulation + * ============================================================================ + */ + +/** Get the type of the item + * + * @param item + * @return The type + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_type cbor_typeof( + const cbor_item_t *item); /* Will be inlined iff link-time opt is enabled */ + +/* Standard CBOR Major item types */ + +/** Does the item have the appropriate major type? + * @param item the item + * @return Is the item an #CBOR_TYPE_UINT? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_isa_uint(const cbor_item_t *item); + +/** Does the item have the appropriate major type? + * @param item the item + * @return Is the item a #CBOR_TYPE_NEGINT? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_isa_negint(const cbor_item_t *item); + +/** Does the item have the appropriate major type? + * @param item the item + * @return Is the item a #CBOR_TYPE_BYTESTRING? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_isa_bytestring(const cbor_item_t *item); + +/** Does the item have the appropriate major type? + * @param item the item + * @return Is the item a #CBOR_TYPE_STRING? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_isa_string(const cbor_item_t *item); + +/** Does the item have the appropriate major type? + * @param item the item + * @return Is the item an #CBOR_TYPE_ARRAY? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_isa_array(const cbor_item_t *item); + +/** Does the item have the appropriate major type? + * @param item the item + * @return Is the item a #CBOR_TYPE_MAP? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_isa_map(const cbor_item_t *item); + +/** Does the item have the appropriate major type? + * @param item the item + * @return Is the item a #CBOR_TYPE_TAG? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_isa_tag(const cbor_item_t *item); + +/** Does the item have the appropriate major type? + * @param item the item + * @return Is the item a #CBOR_TYPE_FLOAT_CTRL? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_isa_float_ctrl(const cbor_item_t *item); + +/* Practical types with respect to their semantics (but not tag values) */ + +/** Is the item an integer, either positive or negative? + * @param item the item + * @return Is the item an integer, either positive or negative? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_is_int(const cbor_item_t *item); + +/** Is the item an a floating point number? + * @param item the item + * @return Is the item a floating point number? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_is_float(const cbor_item_t *item); + +/** Is the item an a boolean? + * @param item the item + * @return Is the item a boolean? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_is_bool(const cbor_item_t *item); + +/** Does this item represent `null` + * + * \rst + * .. warning:: This is in no way related to the value of the pointer. Passing a + * null pointer will most likely result in a crash. + * \endrst + * + * @param item the item + * @return Is the item (CBOR logical) null? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_is_null(const cbor_item_t *item); + +/** Does this item represent `undefined` + * + * \rst + * .. warning:: Care must be taken to distinguish nulls and undefined values in + * C. + * \endrst + * + * @param item the item + * @return Is the item (CBOR logical) undefined? + */ +_CBOR_NODISCARD +CBOR_EXPORT bool cbor_is_undef(const cbor_item_t *item); + +/* + * ============================================================================ + * Memory management + * ============================================================================ + */ + +/** Increases the item's reference count by one + * + * Constant complexity; items referring to this one or items being referred to + * are not updated. + * + * This function can be used to extend reference counting to client code. + * + * @param item Reference to an item + * @return The input \p item + */ +CBOR_EXPORT cbor_item_t *cbor_incref(cbor_item_t *item); + +/** Decreases the item's reference count by one, deallocating the item if needed + * + * In case the item is deallocated, the reference count of all items this item + * references will also be #cbor_decref 'ed recursively. + * + * @param item Reference to an item. Will be set to `NULL` if deallocated + */ +CBOR_EXPORT void cbor_decref(cbor_item_t **item); + +/** Decreases the item's reference count by one, deallocating the item if needed + * + * Convenience wrapper for #cbor_decref when its set-to-null behavior is not + * needed + * + * @param item Reference to an item + */ +CBOR_EXPORT void cbor_intermediate_decref(cbor_item_t *item); + +/** Get the item's reference count + * + * \rst + * .. warning:: This does *not* account for transitive references. + * \endrst + * + * @todo Add some inline examples for reference counting + * + * @param item the item + * @return the reference count + */ +_CBOR_NODISCARD +CBOR_EXPORT size_t cbor_refcount(const cbor_item_t *item); + +/** Provides CPP-like move construct + * + * Decreases the reference count by one, but does not deallocate the item even + * if its refcount reaches zero. This is useful for passing intermediate values + * to functions that increase reference count. Should only be used with + * functions that `incref` their arguments. + * + * \rst + * .. warning:: If the item is moved without correctly increasing the reference + * count afterwards, the memory will be leaked. + * \endrst + * + * @param item Reference to an item + * @return the item with reference count decreased by one + */ +_CBOR_NODISCARD +CBOR_EXPORT cbor_item_t *cbor_move(cbor_item_t *item); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_COMMON_H diff --git a/source/external/libcbor/cbor/configuration.h b/source/external/libcbor/cbor/configuration.h new file mode 100644 index 000000000..83fe90bd3 --- /dev/null +++ b/source/external/libcbor/cbor/configuration.h @@ -0,0 +1,46 @@ +/** + * DO NOT DIRECTLY MODIFY THIS FILE: + * + * The code in this file is generated from scripts/import_libcbor.py + * and any modifications should be in there. + */ + +#ifndef LIBCBOR_CONFIGURATION_H +#define LIBCBOR_CONFIGURATION_H + +#define CBOR_MAJOR_VERSION 0 +#define CBOR_MINOR_VERSION 11 +#define CBOR_PATCH_VERSION 0 + +#define CBOR_BUFFER_GROWTH 2 +#define CBOR_MAX_STACK_SIZE 2048 +#define CBOR_PRETTY_PRINTER 1 + +#if defined(_MSC_VER) +# define CBOR_RESTRICT_SPECIFIER +#else +# define CBOR_RESTRICT_SPECIFIER restrict +#endif + +#define CBOR_INLINE_SPECIFIER + +/* Ignore the compiler warnings for libcbor. */ +#ifdef _MSC_VER +# pragma warning(disable : 4028) +# pragma warning(disable : 4715) +# pragma warning(disable : 4232) +# pragma warning(disable : 4068) +# pragma warning(disable : 4244) +# pragma warning(disable : 4701) +# pragma warning(disable : 4703) +#endif + +#ifdef __clang__ +# pragma clang diagnostic ignored "-Wreturn-type" +#elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wreturn-type" +# pragma GCC diagnostic ignored "-Wunknown-pragmas" +# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + +#endif // LIBCBOR_CONFIGURATION_H diff --git a/source/external/libcbor/cbor/data.h b/source/external/libcbor/cbor/data.h new file mode 100644 index 000000000..a12e92f20 --- /dev/null +++ b/source/external/libcbor/cbor/data.h @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_DATA_H +#define LIBCBOR_DATA_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef const unsigned char* cbor_data; +typedef unsigned char* cbor_mutable_data; + +/** Specifies the Major type of ::cbor_item_t */ +typedef enum cbor_type { + CBOR_TYPE_UINT /** 0 - positive integers */ + , + CBOR_TYPE_NEGINT /** 1 - negative integers*/ + , + CBOR_TYPE_BYTESTRING /** 2 - byte strings */ + , + CBOR_TYPE_STRING /** 3 - strings */ + , + CBOR_TYPE_ARRAY /** 4 - arrays */ + , + CBOR_TYPE_MAP /** 5 - maps */ + , + CBOR_TYPE_TAG /** 6 - tags */ + , + CBOR_TYPE_FLOAT_CTRL /** 7 - decimals and special values (true, false, nil, + ...) */ +} cbor_type; + +/** Possible decoding errors */ +typedef enum { + CBOR_ERR_NONE, + CBOR_ERR_NOTENOUGHDATA, + CBOR_ERR_NODATA, + // TODO: Should be "malformed" or at least "malformatted". Retained for + // backwards compatibility. + CBOR_ERR_MALFORMATED, + CBOR_ERR_MEMERROR /** Memory error - item allocation failed. Is it too big for + your allocator? */ + , + CBOR_ERR_SYNTAXERROR /** Stack parsing algorithm failed */ +} cbor_error_code; + +/** Possible widths of #CBOR_TYPE_UINT items */ +typedef enum { + CBOR_INT_8, + CBOR_INT_16, + CBOR_INT_32, + CBOR_INT_64 +} cbor_int_width; + +/** Possible widths of #CBOR_TYPE_FLOAT_CTRL items */ +typedef enum { + CBOR_FLOAT_0 /** Internal use - ctrl and special values */ + , + CBOR_FLOAT_16 /** Half float */ + , + CBOR_FLOAT_32 /** Single float */ + , + CBOR_FLOAT_64 /** Double */ +} cbor_float_width; + +/** Metadata for dynamically sized types */ +typedef enum { + _CBOR_METADATA_DEFINITE, + _CBOR_METADATA_INDEFINITE +} _cbor_dst_metadata; + +/** Semantic mapping for CTRL simple values */ +typedef enum { + CBOR_CTRL_NONE = 0, + CBOR_CTRL_FALSE = 20, + CBOR_CTRL_TRUE = 21, + CBOR_CTRL_NULL = 22, + CBOR_CTRL_UNDEF = 23 +} _cbor_ctrl; + +// Metadata items use size_t (instead of uint64_t) because items in memory take +// up at least 1B per entry or string byte, so if size_t is narrower than +// uint64_t, we wouldn't be able to create them in the first place and can save +// some space. + +/** Integers specific metadata */ +struct _cbor_int_metadata { + cbor_int_width width; +}; + +/** Bytestrings specific metadata */ +struct _cbor_bytestring_metadata { + size_t length; + _cbor_dst_metadata type; +}; + +/** Strings specific metadata */ +struct _cbor_string_metadata { + size_t length; + size_t codepoint_count; /* Sum of chunks' codepoint_counts for indefinite + strings */ + _cbor_dst_metadata type; +}; + +/** Arrays specific metadata */ +struct _cbor_array_metadata { + size_t allocated; + size_t end_ptr; + _cbor_dst_metadata type; +}; + +/** Maps specific metadata */ +struct _cbor_map_metadata { + size_t allocated; + size_t end_ptr; + _cbor_dst_metadata type; +}; + +/** Arrays specific metadata + * + * The pointer is included - cbor_item_metadata is + * 2 * sizeof(size_t) + sizeof(_cbor_string_type_metadata), + * lets use the space + */ +struct _cbor_tag_metadata { + struct cbor_item_t* tagged_item; + uint64_t value; +}; + +/** Floats specific metadata - includes CTRL values */ +struct _cbor_float_ctrl_metadata { + cbor_float_width width; + uint8_t ctrl; +}; + +/** Raw memory casts helper */ +union _cbor_float_helper { + float as_float; + uint32_t as_uint; +}; + +/** Raw memory casts helper */ +union _cbor_double_helper { + double as_double; + uint64_t as_uint; +}; + +/** Union of metadata across all possible types - discriminated in #cbor_item_t + */ +union cbor_item_metadata { + struct _cbor_int_metadata int_metadata; + struct _cbor_bytestring_metadata bytestring_metadata; + struct _cbor_string_metadata string_metadata; + struct _cbor_array_metadata array_metadata; + struct _cbor_map_metadata map_metadata; + struct _cbor_tag_metadata tag_metadata; + struct _cbor_float_ctrl_metadata float_ctrl_metadata; +}; + +/** The item handle */ +typedef struct cbor_item_t { + /** Discriminated by type */ + union cbor_item_metadata metadata; + /** Reference count - initialize to 0 */ + size_t refcount; + /** Major type discriminator */ + cbor_type type; + /** Raw data block - interpretation depends on metadata */ + unsigned char* data; +} cbor_item_t; + +/** Defines cbor_item_t#data structure for indefinite strings and bytestrings + * + * Used to cast the raw representation for a sane manipulation + */ +struct cbor_indefinite_string_data { + size_t chunk_count; + size_t chunk_capacity; + cbor_item_t** chunks; +}; + +/** High-level decoding error */ +struct cbor_error { + /** Approximate position */ + size_t position; + /** Description */ + cbor_error_code code; +}; + +/** Simple pair of items for use in maps */ +struct cbor_pair { + cbor_item_t *key, *value; +}; + +/** High-level decoding result */ +struct cbor_load_result { + /** Error indicator */ + struct cbor_error error; + /** Number of bytes read */ + size_t read; +}; + +/** Streaming decoder result - status */ +enum cbor_decoder_status { + /** Decoding finished successfully (a callback has been invoked) + * + * Note that this does *not* mean that the buffer has been fully decoded; + * there may still be unread bytes for which no callback has been involved. + */ + CBOR_DECODER_FINISHED, + /** Not enough data to invoke a callback */ + // TODO: The name is inconsistent with CBOR_ERR_NOTENOUGHDATA. Retained for + // backwards compatibility. + CBOR_DECODER_NEDATA, + /** Bad data (reserved MTB, malformed value, etc.) */ + CBOR_DECODER_ERROR +}; + +/** Streaming decoder result */ +struct cbor_decoder_result { + /** Input bytes read/consumed + * + * If this is less than the size of input buffer, the client will likely + * resume parsing starting at the next byte (e.g. `buffer + result.read`). + * + * Set to 0 if the #status is not #CBOR_DECODER_FINISHED. + */ + size_t read; + + /** The decoding status */ + enum cbor_decoder_status status; + + /** Number of bytes in the input buffer needed to resume parsing + * + * Set to 0 unless the result status is #CBOR_DECODER_NEDATA. If it is, then: + * - If at least one byte was passed, #required will be set to the minimum + * number of bytes needed to invoke a decoded callback on the current + * prefix. + * + * For example: Attempting to decode a 1B buffer containing `0x19` will + * set #required to 3 as `0x19` signals a 2B integer item, so we need at + * least 3B to continue (the `0x19` MTB byte and two bytes of data needed + * to invoke #cbor_callbacks.uint16). + * + * - If there was no data at all, #read will always be set to 1 + */ + size_t required; +}; + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_DATA_H diff --git a/source/external/libcbor/cbor/encoding.c b/source/external/libcbor/cbor/encoding.c new file mode 100644 index 000000000..9d931d175 --- /dev/null +++ b/source/external/libcbor/cbor/encoding.c @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "encoding.h" +#include "internal/encoders.h" + +size_t cbor_encode_uint8(uint8_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint8(value, buffer, buffer_size, 0x00); +} + +size_t cbor_encode_uint16(uint16_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint16(value, buffer, buffer_size, 0x00); +} + +size_t cbor_encode_uint32(uint32_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint32(value, buffer, buffer_size, 0x00); +} + +size_t cbor_encode_uint64(uint64_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint64(value, buffer, buffer_size, 0x00); +} + +size_t cbor_encode_uint(uint64_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint(value, buffer, buffer_size, 0x00); +} + +size_t cbor_encode_negint8(uint8_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint8(value, buffer, buffer_size, 0x20); +} + +size_t cbor_encode_negint16(uint16_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint16(value, buffer, buffer_size, 0x20); +} + +size_t cbor_encode_negint32(uint32_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint32(value, buffer, buffer_size, 0x20); +} + +size_t cbor_encode_negint64(uint64_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint64(value, buffer, buffer_size, 0x20); +} + +size_t cbor_encode_negint(uint64_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint(value, buffer, buffer_size, 0x20); +} + +size_t cbor_encode_bytestring_start(size_t length, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint((size_t)length, buffer, buffer_size, 0x40); +} + +size_t _cbor_encode_byte(uint8_t value, unsigned char *buffer, + size_t buffer_size) { + if (buffer_size >= 1) { + buffer[0] = value; + return 1; + } else + return 0; +} + +size_t cbor_encode_indef_bytestring_start(unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_byte(0x5F, buffer, buffer_size); +} + +size_t cbor_encode_string_start(size_t length, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint((size_t)length, buffer, buffer_size, 0x60); +} + +size_t cbor_encode_indef_string_start(unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_byte(0x7F, buffer, buffer_size); +} + +size_t cbor_encode_array_start(size_t length, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint((size_t)length, buffer, buffer_size, 0x80); +} + +size_t cbor_encode_indef_array_start(unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_byte(0x9F, buffer, buffer_size); +} + +size_t cbor_encode_map_start(size_t length, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint((size_t)length, buffer, buffer_size, 0xA0); +} + +size_t cbor_encode_indef_map_start(unsigned char *buffer, size_t buffer_size) { + return _cbor_encode_byte(0xBF, buffer, buffer_size); +} + +size_t cbor_encode_tag(uint64_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint(value, buffer, buffer_size, 0xC0); +} + +size_t cbor_encode_bool(bool value, unsigned char *buffer, size_t buffer_size) { + return value ? _cbor_encode_byte(0xF5, buffer, buffer_size) + : _cbor_encode_byte(0xF4, buffer, buffer_size); +} + +size_t cbor_encode_null(unsigned char *buffer, size_t buffer_size) { + return _cbor_encode_byte(0xF6, buffer, buffer_size); +} + +size_t cbor_encode_undef(unsigned char *buffer, size_t buffer_size) { + return _cbor_encode_byte(0xF7, buffer, buffer_size); +} + +size_t cbor_encode_half(float value, unsigned char *buffer, + size_t buffer_size) { + /* Assuming value is normalized */ + uint32_t val = ((union _cbor_float_helper){.as_float = value}).as_uint; + uint16_t res; + uint8_t exp = (uint8_t)((val & 0x7F800000u) >> + 23u); /* 0b0111_1111_1000_0000_0000_0000_0000_0000 */ + uint32_t mant = + val & 0x7FFFFFu; /* 0b0000_0000_0111_1111_1111_1111_1111_1111 */ + if (exp == 0xFF) { /* Infinity or NaNs */ + if (value != value) { + // We discard information bits in half-float NaNs. This is + // not required for the core CBOR protocol (it is only a suggestion in + // Section 3.9). + // See https://github.com/PJK/libcbor/issues/215 + res = (uint16_t)0x007e00; + } else { + // If the mantissa is non-zero, we have a NaN, but those are handled + // above. See + // https://en.wikipedia.org/wiki/Half-precision_floating-point_format + CBOR_ASSERT(mant == 0u); + res = (uint16_t)((val & 0x80000000u) >> 16u | 0x7C00u); + } + } else if (exp == 0x00) { /* Zeroes or subnorms */ + res = (uint16_t)((val & 0x80000000u) >> 16u | mant >> 13u); + } else { /* Normal numbers */ + int8_t logical_exp = (int8_t)(exp - 127); + CBOR_ASSERT(logical_exp == exp - 127); + + // Now we know that 2^exp <= 0 logically + if (logical_exp < -24) { + /* No unambiguous representation exists, this float is not a half float + and is too small to be represented using a half, round off to zero. + Consistent with the reference implementation. */ + res = 0; + } else if (logical_exp < -14) { + /* Offset the remaining decimal places by shifting the significand, the + value is lost. This is an implementation decision that works around the + absence of standard half-float in the language. */ + res = (uint16_t)((val & 0x80000000u) >> 16u) | // Extract sign bit + ((uint16_t)(1u << (24u + logical_exp)) + + (uint16_t)(((mant >> (-logical_exp - 2)) + 1) >> + 1)); // Round half away from zero for simplicity + } else { + res = (uint16_t)((val & 0x80000000u) >> 16u | + ((((uint8_t)logical_exp) + 15u) << 10u) | + (uint16_t)(mant >> 13u)); + } + } + return _cbor_encode_uint16(res, buffer, buffer_size, 0xE0); +} + +size_t cbor_encode_single(float value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint32( + ((union _cbor_float_helper){.as_float = value}).as_uint, buffer, + buffer_size, 0xE0); +} + +size_t cbor_encode_double(double value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint64( + ((union _cbor_double_helper){.as_double = value}).as_uint, buffer, + buffer_size, 0xE0); +} + +size_t cbor_encode_break(unsigned char *buffer, size_t buffer_size) { + return _cbor_encode_byte(0xFF, buffer, buffer_size); +} + +size_t cbor_encode_ctrl(uint8_t value, unsigned char *buffer, + size_t buffer_size) { + return _cbor_encode_uint8(value, buffer, buffer_size, 0xE0); +} diff --git a/source/external/libcbor/cbor/encoding.h b/source/external/libcbor/cbor/encoding.h new file mode 100644 index 000000000..bcc04f8a9 --- /dev/null +++ b/source/external/libcbor/cbor/encoding.h @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_ENCODING_H +#define LIBCBOR_ENCODING_H + +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * All cbor_encode_* methods take 2 or 3 arguments: + * - a logical `value` to encode (except for trivial items such as NULLs) + * - an output `buffer` pointer + * - a `buffer_size` specification + * + * They serialize the `value` into one or more bytes and write the bytes to the + * output `buffer` and return either the number of bytes written, or 0 if the + * `buffer_size` was too small to small to fit the serialized value (in which + * case it is not modified). + */ + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_uint8(uint8_t, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_uint16(uint16_t, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_uint32(uint32_t, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_uint64(uint64_t, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_uint(uint64_t, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_negint8(uint8_t, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_negint16(uint16_t, + unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_negint32(uint32_t, + unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_negint64(uint64_t, + unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_negint(uint64_t, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_bytestring_start(size_t, + unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t +cbor_encode_indef_bytestring_start(unsigned char *, size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_string_start(size_t, + unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t +cbor_encode_indef_string_start(unsigned char *, size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_array_start(size_t, + unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t +cbor_encode_indef_array_start(unsigned char *, size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_map_start(size_t, + unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_indef_map_start(unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_tag(uint64_t, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_bool(bool, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_null(unsigned char *, size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_undef(unsigned char *, size_t); + +/** Encodes a half-precision float + * + * Since there is no native representation or semantics for half floats + * in the language, we use single-precision floats, as every value that + * can be expressed as a half-float can also be expressed as a float. + * + * This however means that not all floats passed to this function can be + * unambiguously encoded. The behavior is as follows: + * - Infinity, NaN are preserved + * - Zero is preserved + * - Denormalized numbers keep their sign bit and 10 most significant bit of + * the significand + * - All other numbers + * - If the logical value of the exponent is < -24, the output is zero + * - If the logical value of the exponent is between -23 and -14, the output + * is cut off to represent the 'magnitude' of the input, by which we + * mean (-1)^{signbit} x 1.0e{exponent}. The value in the significand is + * lost. + * - In all other cases, the sign bit, the exponent, and 10 most significant + * bits of the significand are kept + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_half(float, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_single(float, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_double(double, unsigned char *, + size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_break(unsigned char *, size_t); + +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_encode_ctrl(uint8_t, unsigned char *, + size_t); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_ENCODING_H diff --git a/source/external/libcbor/cbor/floats_ctrls.c b/source/external/libcbor/cbor/floats_ctrls.c new file mode 100644 index 000000000..57bf477d4 --- /dev/null +++ b/source/external/libcbor/cbor/floats_ctrls.c @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "floats_ctrls.h" +#include +#include "assert.h" + +cbor_float_width cbor_float_get_width(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_float_ctrl(item)); + return item->metadata.float_ctrl_metadata.width; +} + +uint8_t cbor_ctrl_value(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_float_ctrl(item)); + CBOR_ASSERT(cbor_float_get_width(item) == CBOR_FLOAT_0); + return item->metadata.float_ctrl_metadata.ctrl; +} + +bool cbor_float_ctrl_is_ctrl(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_float_ctrl(item)); + return cbor_float_get_width(item) == CBOR_FLOAT_0; +} + +float cbor_float_get_float2(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_float(item)); + CBOR_ASSERT(cbor_float_get_width(item) == CBOR_FLOAT_16); + return *(float *)item->data; +} + +float cbor_float_get_float4(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_float(item)); + CBOR_ASSERT(cbor_float_get_width(item) == CBOR_FLOAT_32); + return *(float *)item->data; +} + +double cbor_float_get_float8(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_float(item)); + CBOR_ASSERT(cbor_float_get_width(item) == CBOR_FLOAT_64); + return *(double *)item->data; +} + +double cbor_float_get_float(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_float(item)); + // cppcheck-suppress missingReturn + switch (cbor_float_get_width(item)) { + case CBOR_FLOAT_0: + return NAN; + case CBOR_FLOAT_16: + return cbor_float_get_float2(item); + case CBOR_FLOAT_32: + return cbor_float_get_float4(item); + case CBOR_FLOAT_64: + return cbor_float_get_float8(item); + } +} + +bool cbor_get_bool(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_bool(item)); + return item->metadata.float_ctrl_metadata.ctrl == CBOR_CTRL_TRUE; +} + +void cbor_set_float2(cbor_item_t *item, float value) { + CBOR_ASSERT(cbor_is_float(item)); + CBOR_ASSERT(cbor_float_get_width(item) == CBOR_FLOAT_16); + *((float *)item->data) = value; +} + +void cbor_set_float4(cbor_item_t *item, float value) { + CBOR_ASSERT(cbor_is_float(item)); + CBOR_ASSERT(cbor_float_get_width(item) == CBOR_FLOAT_32); + *((float *)item->data) = value; +} + +void cbor_set_float8(cbor_item_t *item, double value) { + CBOR_ASSERT(cbor_is_float(item)); + CBOR_ASSERT(cbor_float_get_width(item) == CBOR_FLOAT_64); + *((double *)item->data) = value; +} + +void cbor_set_ctrl(cbor_item_t *item, uint8_t value) { + CBOR_ASSERT(cbor_isa_float_ctrl(item)); + CBOR_ASSERT(cbor_float_get_width(item) == CBOR_FLOAT_0); + item->metadata.float_ctrl_metadata.ctrl = value; +} + +void cbor_set_bool(cbor_item_t *item, bool value) { + CBOR_ASSERT(cbor_is_bool(item)); + item->metadata.float_ctrl_metadata.ctrl = + value ? CBOR_CTRL_TRUE : CBOR_CTRL_FALSE; +} + +cbor_item_t *cbor_new_ctrl(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t)); + _CBOR_NOTNULL(item); + + *item = (cbor_item_t){ + .type = CBOR_TYPE_FLOAT_CTRL, + .data = NULL, + .refcount = 1, + .metadata = {.float_ctrl_metadata = {.width = CBOR_FLOAT_0, + .ctrl = CBOR_CTRL_NONE}}}; + return item; +} + +cbor_item_t *cbor_new_float2(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t) + 4); + _CBOR_NOTNULL(item); + + *item = (cbor_item_t){ + .type = CBOR_TYPE_FLOAT_CTRL, + .data = (unsigned char *)item + sizeof(cbor_item_t), + .refcount = 1, + .metadata = {.float_ctrl_metadata = {.width = CBOR_FLOAT_16}}}; + return item; +} + +cbor_item_t *cbor_new_float4(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t) + 4); + _CBOR_NOTNULL(item); + + *item = (cbor_item_t){ + .type = CBOR_TYPE_FLOAT_CTRL, + .data = (unsigned char *)item + sizeof(cbor_item_t), + .refcount = 1, + .metadata = {.float_ctrl_metadata = {.width = CBOR_FLOAT_32}}}; + return item; +} + +cbor_item_t *cbor_new_float8(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t) + 8); + _CBOR_NOTNULL(item); + + *item = (cbor_item_t){ + .type = CBOR_TYPE_FLOAT_CTRL, + .data = (unsigned char *)item + sizeof(cbor_item_t), + .refcount = 1, + .metadata = {.float_ctrl_metadata = {.width = CBOR_FLOAT_64}}}; + return item; +} + +cbor_item_t *cbor_new_null(void) { + cbor_item_t *item = cbor_new_ctrl(); + _CBOR_NOTNULL(item); + cbor_set_ctrl(item, CBOR_CTRL_NULL); + return item; +} + +cbor_item_t *cbor_new_undef(void) { + cbor_item_t *item = cbor_new_ctrl(); + _CBOR_NOTNULL(item); + cbor_set_ctrl(item, CBOR_CTRL_UNDEF); + return item; +} + +cbor_item_t *cbor_build_bool(bool value) { + return cbor_build_ctrl(value ? CBOR_CTRL_TRUE : CBOR_CTRL_FALSE); +} + +cbor_item_t *cbor_build_float2(float value) { + cbor_item_t *item = cbor_new_float2(); + _CBOR_NOTNULL(item); + cbor_set_float2(item, value); + return item; +} + +cbor_item_t *cbor_build_float4(float value) { + cbor_item_t *item = cbor_new_float4(); + _CBOR_NOTNULL(item); + cbor_set_float4(item, value); + return item; +} + +cbor_item_t *cbor_build_float8(double value) { + cbor_item_t *item = cbor_new_float8(); + _CBOR_NOTNULL(item); + cbor_set_float8(item, value); + return item; +} + +cbor_item_t *cbor_build_ctrl(uint8_t value) { + cbor_item_t *item = cbor_new_ctrl(); + _CBOR_NOTNULL(item); + cbor_set_ctrl(item, value); + return item; +} diff --git a/source/external/libcbor/cbor/floats_ctrls.h b/source/external/libcbor/cbor/floats_ctrls.h new file mode 100644 index 000000000..335eab832 --- /dev/null +++ b/source/external/libcbor/cbor/floats_ctrls.h @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_FLOATS_CTRLS_H +#define LIBCBOR_FLOATS_CTRLS_H + +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * ============================================================================ + * Float manipulation + * ============================================================================ + */ + +/** Is this a ctrl value? + * + * @param item A float or ctrl item + * @return Is this a ctrl value? + */ +_CBOR_NODISCARD CBOR_EXPORT bool cbor_float_ctrl_is_ctrl( + const cbor_item_t *item); + +/** Get the float width + * + * @param item A float or ctrl item + * @return The width. + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_float_width +cbor_float_get_width(const cbor_item_t *item); + +/** Get a half precision float + * + * The item must have the corresponding width + * + * @param item A half precision float + * @return half precision value + */ +_CBOR_NODISCARD CBOR_EXPORT float cbor_float_get_float2( + const cbor_item_t *item); + +/** Get a single precision float + * + * The item must have the corresponding width + * + * @param item A single precision float + * @return single precision value + */ +_CBOR_NODISCARD CBOR_EXPORT float cbor_float_get_float4( + const cbor_item_t *item); + +/** Get a double precision float + * + * The item must have the corresponding width + * + * @param item A double precision float + * @return double precision value + */ +_CBOR_NODISCARD CBOR_EXPORT double cbor_float_get_float8( + const cbor_item_t *item); + +/** Get the float value represented as double + * + * Can be used regardless of the width. + * + * @param item Any float + * @return double precision value + */ +_CBOR_NODISCARD CBOR_EXPORT double cbor_float_get_float( + const cbor_item_t *item); + +/** Get value from a boolean ctrl item + * + * @param item A ctrl item + * @return boolean value + */ +_CBOR_NODISCARD CBOR_EXPORT bool cbor_get_bool(const cbor_item_t *item); + +/** Constructs a new ctrl item + * + * The width cannot be changed once the item is created + * + * @return Reference to the new ctrl item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_ctrl(void); + +/** Constructs a new float item + * + * The width cannot be changed once the item is created + * + * @return Reference to the new float item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_float2(void); + +/** Constructs a new float item + * + * The width cannot be changed once the item is created + * + * @return Reference to the new float item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_float4(void); + +/** Constructs a new float item + * + * The width cannot be changed once the item is created + * + * @return Reference to the new float item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_float8(void); + +/** Constructs new null ctrl item + * + * @return Reference to the new null item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_null(void); + +/** Constructs new undef ctrl item + * + * @return Reference to the new undef item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_undef(void); + +/** Constructs new boolean ctrl item + * + * @param value The value to use + * @return Reference to the new boolean item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_bool(bool value); + +/** Assign a control value + * + * \rst + * .. warning:: It is possible to produce an invalid CBOR value by assigning a + * invalid value using this mechanism. Please consult the standard before use. + * \endrst + * + * @param item A ctrl item + * @param value The simple value to assign. Please consult the standard for + * allowed values + */ +CBOR_EXPORT void cbor_set_ctrl(cbor_item_t *item, uint8_t value); + +/** Assign a boolean value to a boolean ctrl item + * + * @param item A ctrl item + * @param value The simple value to assign. + */ +CBOR_EXPORT void cbor_set_bool(cbor_item_t *item, bool value); + +/** Assigns a float value + * + * @param item A half precision float + * @param value The value to assign + */ +CBOR_EXPORT void cbor_set_float2(cbor_item_t *item, float value); + +/** Assigns a float value + * + * @param item A single precision float + * @param value The value to assign + */ +CBOR_EXPORT void cbor_set_float4(cbor_item_t *item, float value); + +/** Assigns a float value + * + * @param item A double precision float + * @param value The value to assign + */ +CBOR_EXPORT void cbor_set_float8(cbor_item_t *item, double value); + +/** Reads the control value + * + * @param item A ctrl item + * @return the simple value + */ +_CBOR_NODISCARD CBOR_EXPORT uint8_t cbor_ctrl_value(const cbor_item_t *item); + +/** Constructs a new float + * + * @param value the value to use + * @return Reference to the new float item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_float2(float value); + +/** Constructs a new float + * + * @param value the value to use + * @return Reference to the new float item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_float4(float value); + +/** Constructs a new float + * + * @param value the value to use + * @return Reference to the new float item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_float8(double value); + +/** Constructs a ctrl item + * + * @param value the value to use + * @return Reference to the new ctrl item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_ctrl(uint8_t value); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_FLOATS_CTRLS_H diff --git a/source/external/libcbor/cbor/internal/builder_callbacks.c b/source/external/libcbor/cbor/internal/builder_callbacks.c new file mode 100644 index 000000000..257cef3ad --- /dev/null +++ b/source/external/libcbor/cbor/internal/builder_callbacks.c @@ -0,0 +1,422 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "builder_callbacks.h" + +#include + +#include "../arrays.h" +#include "../bytestrings.h" +#include "../common.h" +#include "../floats_ctrls.h" +#include "../ints.h" +#include "../maps.h" +#include "../strings.h" +#include "../tags.h" +#include "unicode.h" + +// `_cbor_builder_append` takes ownership of `item`. If adding the item to +// parent container fails, `item` will be deallocated to prevent memory. +void _cbor_builder_append(cbor_item_t *item, + struct _cbor_decoder_context *ctx) { + if (ctx->stack->size == 0) { + /* Top level item */ + ctx->root = item; + return; + } + /* Part of a bigger structure */ + switch (ctx->stack->top->item->type) { + // Handle Arrays and Maps since they can contain subitems of any type. + // Byte/string construction from chunks is handled in the respective chunk + // handlers. + case CBOR_TYPE_ARRAY: { + if (cbor_array_is_definite(ctx->stack->top->item)) { + // We don't need an explicit check for whether the item still belongs + // into this array because if there are extra items, they will cause a + // syntax error when decoded. + CBOR_ASSERT(ctx->stack->top->subitems > 0); + // This should never happen since the definite array should be + // preallocated for the expected number of items. + if (!cbor_array_push(ctx->stack->top->item, item)) { + ctx->creation_failed = true; + cbor_decref(&item); + break; + } + cbor_decref(&item); + ctx->stack->top->subitems--; + if (ctx->stack->top->subitems == 0) { + cbor_item_t *stack_item = ctx->stack->top->item; + _cbor_stack_pop(ctx->stack); + _cbor_builder_append(stack_item, ctx); + } + } else { + /* Indefinite array, don't bother with subitems */ + if (!cbor_array_push(ctx->stack->top->item, item)) { + ctx->creation_failed = true; + } + cbor_decref(&item); + } + break; + } + case CBOR_TYPE_MAP: { + // Handle both definite and indefinite maps the same initially. + // Note: We use 0 and 1 subitems to distinguish between keys and values in + // indefinite items + if (ctx->stack->top->subitems % 2) { + // Odd record, this is a value. + ctx->creation_failed = + !_cbor_map_add_value(ctx->stack->top->item, item); + // Adding a value never fails since the memory is allocated when the + // key is added + CBOR_ASSERT(!ctx->creation_failed); + } else { + // Even record, this is a key. + if (!_cbor_map_add_key(ctx->stack->top->item, item)) { + ctx->creation_failed = true; + cbor_decref(&item); + break; + } + } + cbor_decref(&item); + if (cbor_map_is_definite(ctx->stack->top->item)) { + CBOR_ASSERT(ctx->stack->top->subitems > 0); + ctx->stack->top->subitems--; + if (ctx->stack->top->subitems == 0) { + cbor_item_t *map_entry = ctx->stack->top->item; + _cbor_stack_pop(ctx->stack); + _cbor_builder_append(map_entry, ctx); + } + } else { + ctx->stack->top->subitems ^= + 1; /* Flip the indicator for indefinite items */ + } + break; + } + case CBOR_TYPE_TAG: { + CBOR_ASSERT(ctx->stack->top->subitems == 1); + cbor_tag_set_item(ctx->stack->top->item, item); + cbor_decref(&item); /* Give up on our reference */ + cbor_item_t *tagged_item = ctx->stack->top->item; + _cbor_stack_pop(ctx->stack); + _cbor_builder_append(tagged_item, ctx); + break; + } + // We have an item to append but nothing to append it to. + default: { + cbor_decref(&item); + ctx->syntax_error = true; + } + } +} + +#define CHECK_RES(ctx, res) \ + do { \ + if (res == NULL) { \ + ctx->creation_failed = true; \ + return; \ + } \ + } while (0) + +// Check that the length fits into size_t. If not, we cannot possibly allocate +// the required memory and should fail fast. +#define CHECK_LENGTH(ctx, length) \ + do { \ + if (length > SIZE_MAX) { \ + ctx->creation_failed = true; \ + return; \ + } \ + } while (0) + +#define PUSH_CTX_STACK(ctx, res, subitems) \ + do { \ + if (_cbor_stack_push(ctx->stack, res, subitems) == NULL) { \ + cbor_decref(&res); \ + ctx->creation_failed = true; \ + } \ + } while (0) + +void cbor_builder_uint8_callback(void *context, uint8_t value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_int8(); + CHECK_RES(ctx, res); + cbor_mark_uint(res); + cbor_set_uint8(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_uint16_callback(void *context, uint16_t value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_int16(); + CHECK_RES(ctx, res); + cbor_mark_uint(res); + cbor_set_uint16(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_uint32_callback(void *context, uint32_t value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_int32(); + CHECK_RES(ctx, res); + cbor_mark_uint(res); + cbor_set_uint32(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_uint64_callback(void *context, uint64_t value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_int64(); + CHECK_RES(ctx, res); + cbor_mark_uint(res); + cbor_set_uint64(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_negint8_callback(void *context, uint8_t value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_int8(); + CHECK_RES(ctx, res); + cbor_mark_negint(res); + cbor_set_uint8(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_negint16_callback(void *context, uint16_t value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_int16(); + CHECK_RES(ctx, res); + cbor_mark_negint(res); + cbor_set_uint16(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_negint32_callback(void *context, uint32_t value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_int32(); + CHECK_RES(ctx, res); + cbor_mark_negint(res); + cbor_set_uint32(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_negint64_callback(void *context, uint64_t value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_int64(); + CHECK_RES(ctx, res); + cbor_mark_negint(res); + cbor_set_uint64(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_byte_string_callback(void *context, cbor_data data, + uint64_t length) { + struct _cbor_decoder_context *ctx = context; + CHECK_LENGTH(ctx, length); + unsigned char *new_handle = _cbor_malloc(length); + if (new_handle == NULL) { + ctx->creation_failed = true; + return; + } + + memcpy(new_handle, data, length); + cbor_item_t *new_chunk = cbor_new_definite_bytestring(); + + if (new_chunk == NULL) { + _cbor_free(new_handle); + ctx->creation_failed = true; + return; + } + + cbor_bytestring_set_handle(new_chunk, new_handle, length); + + // If an indef bytestring is on the stack, extend it (if it were closed, it + // would have been popped). Handle any syntax errors upstream. + if (ctx->stack->size > 0 && cbor_isa_bytestring(ctx->stack->top->item) && + cbor_bytestring_is_indefinite(ctx->stack->top->item)) { + if (!cbor_bytestring_add_chunk(ctx->stack->top->item, new_chunk)) { + ctx->creation_failed = true; + } + cbor_decref(&new_chunk); + } else { + _cbor_builder_append(new_chunk, ctx); + } +} + +void cbor_builder_byte_string_start_callback(void *context) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_indefinite_bytestring(); + CHECK_RES(ctx, res); + PUSH_CTX_STACK(ctx, res, 0); +} + +void cbor_builder_string_callback(void *context, cbor_data data, + uint64_t length) { + struct _cbor_decoder_context *ctx = context; + CHECK_LENGTH(ctx, length); + + unsigned char *new_handle = _cbor_malloc(length); + if (new_handle == NULL) { + ctx->creation_failed = true; + return; + } + + memcpy(new_handle, data, length); + cbor_item_t *new_chunk = cbor_new_definite_string(); + if (new_chunk == NULL) { + _cbor_free(new_handle); + ctx->creation_failed = true; + return; + } + cbor_string_set_handle(new_chunk, new_handle, length); + + // If an indef string is on the stack, extend it (if it were closed, it would + // have been popped). Handle any syntax errors upstream. + if (ctx->stack->size > 0 && cbor_isa_string(ctx->stack->top->item) && + cbor_string_is_indefinite(ctx->stack->top->item)) { + if (!cbor_string_add_chunk(ctx->stack->top->item, new_chunk)) { + ctx->creation_failed = true; + } + cbor_decref(&new_chunk); + } else { + _cbor_builder_append(new_chunk, ctx); + } +} + +void cbor_builder_string_start_callback(void *context) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_indefinite_string(); + CHECK_RES(ctx, res); + PUSH_CTX_STACK(ctx, res, 0); +} + +void cbor_builder_array_start_callback(void *context, uint64_t size) { + struct _cbor_decoder_context *ctx = context; + CHECK_LENGTH(ctx, size); + cbor_item_t *res = cbor_new_definite_array(size); + CHECK_RES(ctx, res); + if (size > 0) { + PUSH_CTX_STACK(ctx, res, size); + } else { + _cbor_builder_append(res, ctx); + } +} + +void cbor_builder_indef_array_start_callback(void *context) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_indefinite_array(); + CHECK_RES(ctx, res); + PUSH_CTX_STACK(ctx, res, 0); +} + +void cbor_builder_indef_map_start_callback(void *context) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_indefinite_map(); + CHECK_RES(ctx, res); + PUSH_CTX_STACK(ctx, res, 0); +} + +void cbor_builder_map_start_callback(void *context, uint64_t size) { + struct _cbor_decoder_context *ctx = context; + CHECK_LENGTH(ctx, size); + cbor_item_t *res = cbor_new_definite_map(size); + CHECK_RES(ctx, res); + if (size > 0) { + PUSH_CTX_STACK(ctx, res, size * 2); + } else { + _cbor_builder_append(res, ctx); + } +} + +/** + * Is the (partially constructed) item indefinite? + */ +bool _cbor_is_indefinite(cbor_item_t *item) { + switch (item->type) { + case CBOR_TYPE_BYTESTRING: + return cbor_bytestring_is_indefinite(item); + case CBOR_TYPE_STRING: + return cbor_string_is_indefinite(item); + case CBOR_TYPE_ARRAY: + return cbor_array_is_indefinite(item); + case CBOR_TYPE_MAP: + return cbor_map_is_indefinite(item); + default: + // Should never happen since a non-nested item cannot be on top of the + // stack. + return false; + } +} + +void cbor_builder_indef_break_callback(void *context) { + struct _cbor_decoder_context *ctx = context; + /* There must be an item to break out of*/ + if (ctx->stack->size > 0) { + cbor_item_t *item = ctx->stack->top->item; + if (_cbor_is_indefinite( + item) && /* Only indefinite items can be terminated by 0xFF */ + /* Special case: we cannot append up if an indefinite map is incomplete + (we are expecting a value). */ + (item->type != CBOR_TYPE_MAP || ctx->stack->top->subitems % 2 == 0)) { + _cbor_stack_pop(ctx->stack); + _cbor_builder_append(item, ctx); + return; + } + } + + ctx->syntax_error = true; +} + +void cbor_builder_float2_callback(void *context, float value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_float2(); + CHECK_RES(ctx, res); + cbor_set_float2(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_float4_callback(void *context, float value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_float4(); + CHECK_RES(ctx, res); + cbor_set_float4(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_float8_callback(void *context, double value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_float8(); + CHECK_RES(ctx, res); + cbor_set_float8(res, value); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_null_callback(void *context) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_null(); + CHECK_RES(ctx, res); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_undefined_callback(void *context) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_undef(); + CHECK_RES(ctx, res); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_boolean_callback(void *context, bool value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_build_bool(value); + CHECK_RES(ctx, res); + _cbor_builder_append(res, ctx); +} + +void cbor_builder_tag_callback(void *context, uint64_t value) { + struct _cbor_decoder_context *ctx = context; + cbor_item_t *res = cbor_new_tag(value); + CHECK_RES(ctx, res); + PUSH_CTX_STACK(ctx, res, 1); +} diff --git a/source/external/libcbor/cbor/internal/builder_callbacks.h b/source/external/libcbor/cbor/internal/builder_callbacks.h new file mode 100644 index 000000000..7893960e4 --- /dev/null +++ b/source/external/libcbor/cbor/internal/builder_callbacks.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_BUILDER_CALLBACKS_H +#define LIBCBOR_BUILDER_CALLBACKS_H + +#include "../callbacks.h" +#include "cbor/common.h" +#include "stack.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** High-level decoding context */ +struct _cbor_decoder_context { + /** Callback creating the last item has failed */ + bool creation_failed; + /** Stack expectation mismatch */ + bool syntax_error; + cbor_item_t *root; + struct _cbor_stack *stack; +}; + +/** Internal helper: Append item to the top of the stack while handling errors. + */ +void _cbor_builder_append(cbor_item_t *item, struct _cbor_decoder_context *ctx); + +void cbor_builder_uint8_callback(void *, uint8_t); + +void cbor_builder_uint16_callback(void *, uint16_t); + +void cbor_builder_uint32_callback(void *, uint32_t); + +void cbor_builder_uint64_callback(void *, uint64_t); + +void cbor_builder_negint8_callback(void *, uint8_t); + +void cbor_builder_negint16_callback(void *, uint16_t); + +void cbor_builder_negint32_callback(void *, uint32_t); + +void cbor_builder_negint64_callback(void *, uint64_t); + +void cbor_builder_string_callback(void *, cbor_data, uint64_t); + +void cbor_builder_string_start_callback(void *); + +void cbor_builder_byte_string_callback(void *, cbor_data, uint64_t); + +void cbor_builder_byte_string_start_callback(void *); + +void cbor_builder_array_start_callback(void *, uint64_t); + +void cbor_builder_indef_array_start_callback(void *); + +void cbor_builder_map_start_callback(void *, uint64_t); + +void cbor_builder_indef_map_start_callback(void *); + +void cbor_builder_tag_callback(void *, uint64_t); + +void cbor_builder_float2_callback(void *, float); + +void cbor_builder_float4_callback(void *, float); + +void cbor_builder_float8_callback(void *, double); + +void cbor_builder_null_callback(void *); + +void cbor_builder_undefined_callback(void *); + +void cbor_builder_boolean_callback(void *, bool); + +void cbor_builder_indef_break_callback(void *); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_BUILDER_CALLBACKS_H diff --git a/source/external/libcbor/cbor/internal/encoders.c b/source/external/libcbor/cbor/internal/encoders.c new file mode 100644 index 000000000..49d4d7f33 --- /dev/null +++ b/source/external/libcbor/cbor/internal/encoders.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "encoders.h" +#include + +size_t _cbor_encode_uint8(uint8_t value, unsigned char *buffer, + size_t buffer_size, uint8_t offset) { + if (value <= 23) { + if (buffer_size >= 1) { + buffer[0] = value + offset; + return 1; + } + } else { + if (buffer_size >= 2) { + buffer[0] = 0x18 + offset; + buffer[1] = value; + return 2; + } + } + return 0; +} + +size_t _cbor_encode_uint16(uint16_t value, unsigned char *buffer, + size_t buffer_size, uint8_t offset) { + if (buffer_size >= 3) { + buffer[0] = 0x19 + offset; + +#ifdef IS_BIG_ENDIAN + memcpy(buffer + 1, &value, 2); +#else + buffer[1] = (unsigned char)(value >> 8); + buffer[2] = (unsigned char)value; +#endif + + return 3; + } else + return 0; +} + +size_t _cbor_encode_uint32(uint32_t value, unsigned char *buffer, + size_t buffer_size, uint8_t offset) { + if (buffer_size >= 5) { + buffer[0] = 0x1A + offset; + +#ifdef IS_BIG_ENDIAN + memcpy(buffer + 1, &value, 4); +#else + buffer[1] = (unsigned char)(value >> 24); + buffer[2] = (unsigned char)(value >> 16); + buffer[3] = (unsigned char)(value >> 8); + buffer[4] = (unsigned char)value; +#endif + + return 5; + } else + return 0; +} + +size_t _cbor_encode_uint64(uint64_t value, unsigned char *buffer, + size_t buffer_size, uint8_t offset) { + if (buffer_size >= 9) { + buffer[0] = 0x1B + offset; + +#ifdef IS_BIG_ENDIAN + memcpy(buffer + 1, &value, 8); +#else + buffer[1] = (unsigned char)(value >> 56); + buffer[2] = (unsigned char)(value >> 48); + buffer[3] = (unsigned char)(value >> 40); + buffer[4] = (unsigned char)(value >> 32); + buffer[5] = (unsigned char)(value >> 24); + buffer[6] = (unsigned char)(value >> 16); + buffer[7] = (unsigned char)(value >> 8); + buffer[8] = (unsigned char)value; +#endif + + return 9; + } else + return 0; +} + +size_t _cbor_encode_uint(uint64_t value, unsigned char *buffer, + size_t buffer_size, uint8_t offset) { + if (value <= UINT16_MAX) + if (value <= UINT8_MAX) + return _cbor_encode_uint8((uint8_t)value, buffer, buffer_size, offset); + else + return _cbor_encode_uint16((uint16_t)value, buffer, buffer_size, offset); + else if (value <= UINT32_MAX) + return _cbor_encode_uint32((uint32_t)value, buffer, buffer_size, offset); + else + return _cbor_encode_uint64((uint64_t)value, buffer, buffer_size, offset); +} diff --git a/source/external/libcbor/cbor/internal/encoders.h b/source/external/libcbor/cbor/internal/encoders.h new file mode 100644 index 000000000..7eadb7121 --- /dev/null +++ b/source/external/libcbor/cbor/internal/encoders.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_ENCODERS_H +#define LIBCBOR_ENCODERS_H + +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +_CBOR_NODISCARD +size_t _cbor_encode_uint8(uint8_t value, unsigned char *buffer, + size_t buffer_size, uint8_t offset); + +_CBOR_NODISCARD +size_t _cbor_encode_uint16(uint16_t value, unsigned char *buffer, + size_t buffer_size, uint8_t offset); + +_CBOR_NODISCARD +size_t _cbor_encode_uint32(uint32_t value, unsigned char *buffer, + size_t buffer_size, uint8_t offset); + +_CBOR_NODISCARD +size_t _cbor_encode_uint64(uint64_t value, unsigned char *buffer, + size_t buffer_size, uint8_t offset); + +_CBOR_NODISCARD +size_t _cbor_encode_uint(uint64_t value, unsigned char *buffer, + size_t buffer_size, uint8_t offset); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_ENCODERS_H diff --git a/source/external/libcbor/cbor/internal/loaders.c b/source/external/libcbor/cbor/internal/loaders.c new file mode 100644 index 000000000..cfa173de7 --- /dev/null +++ b/source/external/libcbor/cbor/internal/loaders.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "loaders.h" +#include +#include + +uint8_t _cbor_load_uint8(cbor_data source) { return (uint8_t)*source; } + +uint16_t _cbor_load_uint16(const unsigned char *source) { +#ifdef IS_BIG_ENDIAN + uint16_t result; + memcpy(&result, source, 2); + return result; +#else + return ((uint16_t) * (source + 0) << 8) + (uint8_t) * (source + 1); +#endif +} + +uint32_t _cbor_load_uint32(const unsigned char *source) { +#ifdef IS_BIG_ENDIAN + uint32_t result; + memcpy(&result, source, 4); + return result; +#else + return ((uint32_t) * (source + 0) << 0x18) + + ((uint32_t) * (source + 1) << 0x10) + + ((uint16_t) * (source + 2) << 0x08) + (uint8_t) * (source + 3); +#endif +} + +uint64_t _cbor_load_uint64(const unsigned char *source) { +#ifdef IS_BIG_ENDIAN + uint64_t result; + memcpy(&result, source, 8); + return result; +#else + return ((uint64_t) * (source + 0) << 0x38) + + ((uint64_t) * (source + 1) << 0x30) + + ((uint64_t) * (source + 2) << 0x28) + + ((uint64_t) * (source + 3) << 0x20) + + ((uint32_t) * (source + 4) << 0x18) + + ((uint32_t) * (source + 5) << 0x10) + + ((uint16_t) * (source + 6) << 0x08) + (uint8_t) * (source + 7); +#endif +} + +/* As per https://www.rfc-editor.org/rfc/rfc8949.html#name-half-precision */ +float _cbor_decode_half(unsigned char *halfp) { + int half = (halfp[0] << 8) + halfp[1]; + int exp = (half >> 10) & 0x1f; + int mant = half & 0x3ff; + double val; + if (exp == 0) + val = ldexp(mant, -24); + else if (exp != 31) + val = ldexp(mant + 1024, exp - 25); + else + val = mant == 0 ? INFINITY : NAN; + return (float)(half & 0x8000 ? -val : val); +} + +float _cbor_load_half(cbor_data source) { + /* Discard const */ + return _cbor_decode_half((unsigned char *)source); +} + +float _cbor_load_float(cbor_data source) { + union _cbor_float_helper helper = {.as_uint = _cbor_load_uint32(source)}; + return helper.as_float; +} + +double _cbor_load_double(cbor_data source) { + union _cbor_double_helper helper = {.as_uint = _cbor_load_uint64(source)}; + return helper.as_double; +} diff --git a/source/external/libcbor/cbor/internal/loaders.h b/source/external/libcbor/cbor/internal/loaders.h new file mode 100644 index 000000000..ce37563a3 --- /dev/null +++ b/source/external/libcbor/cbor/internal/loaders.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_LOADERS_H +#define LIBCBOR_LOADERS_H + +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Read the given uint from the given location, no questions asked */ +_CBOR_NODISCARD +uint8_t _cbor_load_uint8(const unsigned char *source); + +_CBOR_NODISCARD +uint16_t _cbor_load_uint16(const unsigned char *source); + +_CBOR_NODISCARD +uint32_t _cbor_load_uint32(const unsigned char *source); + +_CBOR_NODISCARD +uint64_t _cbor_load_uint64(const unsigned char *source); + +_CBOR_NODISCARD +float _cbor_load_half(cbor_data source); + +_CBOR_NODISCARD +float _cbor_load_float(cbor_data source); + +_CBOR_NODISCARD +double _cbor_load_double(cbor_data source); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_LOADERS_H diff --git a/source/external/libcbor/cbor/internal/memory_utils.c b/source/external/libcbor/cbor/internal/memory_utils.c new file mode 100644 index 000000000..bbea63cb9 --- /dev/null +++ b/source/external/libcbor/cbor/internal/memory_utils.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "memory_utils.h" +#include "cbor/common.h" + +// TODO: Consider builtins +// (https://gcc.gnu.org/onlinedocs/gcc/Integer-Overflow-Builtins.html) + +/** Highest on bit position */ +size_t _cbor_highest_bit(size_t number) { + size_t bit = 0; + while (number != 0) { + bit++; + number >>= 1; + } + + return bit; +} + +bool _cbor_safe_to_multiply(size_t a, size_t b) { + if (a <= 1 || b <= 1) return true; + return _cbor_highest_bit(a) + _cbor_highest_bit(b) <= sizeof(size_t) * 8; +} + +bool _cbor_safe_to_add(size_t a, size_t b) { + // Unsigned integer overflow doesn't constitute UB + size_t sum = a + b; + return sum >= a && sum >= b; +} + +size_t _cbor_safe_signaling_add(size_t a, size_t b) { + if (a == 0 || b == 0) return 0; + if (_cbor_safe_to_add(a, b)) return a + b; + return 0; +} + +void* _cbor_alloc_multiple(size_t item_size, size_t item_count) { + if (_cbor_safe_to_multiply(item_size, item_count)) { + return _cbor_malloc(item_size * item_count); + } else { + return NULL; + } +} + +void* _cbor_realloc_multiple(void* pointer, size_t item_size, + size_t item_count) { + if (_cbor_safe_to_multiply(item_size, item_count)) { + return _cbor_realloc(pointer, item_size * item_count); + } else { + return NULL; + } +} diff --git a/source/external/libcbor/cbor/internal/memory_utils.h b/source/external/libcbor/cbor/internal/memory_utils.h new file mode 100644 index 000000000..696f67800 --- /dev/null +++ b/source/external/libcbor/cbor/internal/memory_utils.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_MEMORY_UTILS_H +#define LIBCBOR_MEMORY_UTILS_H + +#include +#include + +#include "cbor/common.h" + +/** Can `a` and `b` be multiplied without overflowing size_t? */ +_CBOR_NODISCARD +bool _cbor_safe_to_multiply(size_t a, size_t b); + +/** Can `a` and `b` be added without overflowing size_t? */ +_CBOR_NODISCARD +bool _cbor_safe_to_add(size_t a, size_t b); + +/** Adds `a` and `b`, propagating zeros and returning 0 on overflow. */ +_CBOR_NODISCARD +size_t _cbor_safe_signaling_add(size_t a, size_t b); + +/** Overflow-proof contiguous array allocation + * + * @param item_size + * @param item_count + * @return Region of item_size * item_count bytes, or NULL if the total size + * overflows size_t or the underlying allocator failed + */ +void* _cbor_alloc_multiple(size_t item_size, size_t item_count); + +/** Overflow-proof contiguous array reallocation + * + * This implements the OpenBSD `reallocarray` functionality. + * + * @param pointer + * @param item_size + * @param item_count + * @return Realloc'd of item_size * item_count bytes, or NULL if the total size + * overflows size_t or the underlying allocator failed + */ +void* _cbor_realloc_multiple(void* pointer, size_t item_size, + size_t item_count); + +#endif // LIBCBOR_MEMORY_UTILS_H diff --git a/source/external/libcbor/cbor/internal/stack.c b/source/external/libcbor/cbor/internal/stack.c new file mode 100644 index 000000000..2db03cbbf --- /dev/null +++ b/source/external/libcbor/cbor/internal/stack.c @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "stack.h" + +struct _cbor_stack _cbor_stack_init(void) { + return (struct _cbor_stack){.top = NULL, .size = 0}; +} + +void _cbor_stack_pop(struct _cbor_stack *stack) { + struct _cbor_stack_record *top = stack->top; + stack->top = stack->top->lower; + _cbor_free(top); + stack->size--; +} + +struct _cbor_stack_record *_cbor_stack_push(struct _cbor_stack *stack, + cbor_item_t *item, + size_t subitems) { + if (stack->size == CBOR_MAX_STACK_SIZE) return NULL; + struct _cbor_stack_record *new_top = + _cbor_malloc(sizeof(struct _cbor_stack_record)); + if (new_top == NULL) return NULL; + + *new_top = (struct _cbor_stack_record){stack->top, item, subitems}; + stack->top = new_top; + stack->size++; + return new_top; +} diff --git a/source/external/libcbor/cbor/internal/stack.h b/source/external/libcbor/cbor/internal/stack.h new file mode 100644 index 000000000..cf2206b40 --- /dev/null +++ b/source/external/libcbor/cbor/internal/stack.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_STACK_H +#define LIBCBOR_STACK_H + +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Simple stack record for the parser */ +struct _cbor_stack_record { + /** Pointer to the parent stack frame */ + struct _cbor_stack_record *lower; + /** Item under construction */ + cbor_item_t *item; + /** + * How many outstanding subitems are expected. + * + * For example, when we see a new definite array, `subitems` is initialized to + * the array length. With every item added, the counter is decreased. When it + * reaches zero, the stack is popped and the complete item is propagated + * upwards. + */ + size_t subitems; +}; + +/** Stack handle - contents and size */ +struct _cbor_stack { + struct _cbor_stack_record *top; + size_t size; +}; + +_CBOR_NODISCARD +struct _cbor_stack _cbor_stack_init(void); + +void _cbor_stack_pop(struct _cbor_stack *); + +_CBOR_NODISCARD +struct _cbor_stack_record *_cbor_stack_push(struct _cbor_stack *, cbor_item_t *, + size_t); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_STACK_H diff --git a/source/external/libcbor/cbor/internal/unicode.c b/source/external/libcbor/cbor/internal/unicode.c new file mode 100644 index 000000000..f87b746a3 --- /dev/null +++ b/source/external/libcbor/cbor/internal/unicode.c @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "unicode.h" +#include + +#define UTF8_ACCEPT 0 +#define UTF8_REJECT 1 + +static const uint8_t utf8d[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 00..1f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 20..3f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 40..5f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 60..7f */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, /* 80..9f */ + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, /* a0..bf */ + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* c0..df */ + 0xa, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, + 0x3, 0x3, 0x4, 0x3, 0x3, /* e0..ef */ + 0xb, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, + 0x8, 0x8, 0x8, 0x8, 0x8, /* f0..ff */ + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, + 0x6, 0x1, 0x1, 0x1, 0x1, /* s0..s0 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, + 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, /* s1..s2 */ + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, /* s3..s4 */ + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, /* s5..s6 */ + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, + 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* s7..s8 */ +}; + +/* Copyright of this function: (c) 2008-2009 Bjoern Hoehrmann + * */ +/* See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. */ +uint32_t _cbor_unicode_decode(uint32_t* state, uint32_t* codep, uint32_t byte) { + uint32_t type = utf8d[byte]; + + *codep = (*state != UTF8_ACCEPT) ? (byte & 0x3fu) | (*codep << 6) + : (0xff >> type) & (byte); + + *state = utf8d[256 + *state * 16 + type]; + return *state; +} + +size_t _cbor_unicode_codepoint_count(cbor_data source, size_t source_length, + struct _cbor_unicode_status* status) { + *status = + (struct _cbor_unicode_status){.location = 0, .status = _CBOR_UNICODE_OK}; + uint32_t codepoint, state = UTF8_ACCEPT, res; + size_t pos = 0, count = 0; + + for (; pos < source_length; pos++) { + res = _cbor_unicode_decode(&state, &codepoint, source[pos]); + + if (res == UTF8_ACCEPT) { + count++; + } else if (res == UTF8_REJECT) { + goto error; + } + } + + /* Unfinished multibyte codepoint */ + if (state != UTF8_ACCEPT) goto error; + + return count; + +error: + *status = (struct _cbor_unicode_status){.location = pos, + .status = _CBOR_UNICODE_BADCP}; + return 0; +} diff --git a/source/external/libcbor/cbor/internal/unicode.h b/source/external/libcbor/cbor/internal/unicode.h new file mode 100644 index 000000000..81d03d007 --- /dev/null +++ b/source/external/libcbor/cbor/internal/unicode.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_UNICODE_H +#define LIBCBOR_UNICODE_H + +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +enum _cbor_unicode_status_error { _CBOR_UNICODE_OK, _CBOR_UNICODE_BADCP }; + +/** Signals unicode validation error and possibly its location */ +struct _cbor_unicode_status { + enum _cbor_unicode_status_error status; + size_t location; +}; + +_CBOR_NODISCARD +size_t _cbor_unicode_codepoint_count(cbor_data source, size_t source_length, + struct _cbor_unicode_status* status); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_UNICODE_H diff --git a/source/external/libcbor/cbor/ints.c b/source/external/libcbor/cbor/ints.c new file mode 100644 index 000000000..b4d035a18 --- /dev/null +++ b/source/external/libcbor/cbor/ints.c @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "ints.h" + +cbor_int_width cbor_int_get_width(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_int(item)); + return item->metadata.int_metadata.width; +} + +uint8_t cbor_get_uint8(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_int(item)); + CBOR_ASSERT(cbor_int_get_width(item) == CBOR_INT_8); + return *item->data; +} + +uint16_t cbor_get_uint16(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_int(item)); + CBOR_ASSERT(cbor_int_get_width(item) == CBOR_INT_16); + return *(uint16_t *)item->data; +} + +uint32_t cbor_get_uint32(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_int(item)); + CBOR_ASSERT(cbor_int_get_width(item) == CBOR_INT_32); + return *(uint32_t *)item->data; +} + +uint64_t cbor_get_uint64(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_int(item)); + CBOR_ASSERT(cbor_int_get_width(item) == CBOR_INT_64); + return *(uint64_t *)item->data; +} + +uint64_t cbor_get_int(const cbor_item_t *item) { + CBOR_ASSERT(cbor_is_int(item)); + // cppcheck-suppress missingReturn + switch (cbor_int_get_width(item)) { + case CBOR_INT_8: + return cbor_get_uint8(item); + case CBOR_INT_16: + return cbor_get_uint16(item); + case CBOR_INT_32: + return cbor_get_uint32(item); + case CBOR_INT_64: + return cbor_get_uint64(item); + } +} + +void cbor_set_uint8(cbor_item_t *item, uint8_t value) { + CBOR_ASSERT(cbor_is_int(item)); + CBOR_ASSERT(cbor_int_get_width(item) == CBOR_INT_8); + *item->data = value; +} + +void cbor_set_uint16(cbor_item_t *item, uint16_t value) { + CBOR_ASSERT(cbor_is_int(item)); + CBOR_ASSERT(cbor_int_get_width(item) == CBOR_INT_16); + *(uint16_t *)item->data = value; +} + +void cbor_set_uint32(cbor_item_t *item, uint32_t value) { + CBOR_ASSERT(cbor_is_int(item)); + CBOR_ASSERT(cbor_int_get_width(item) == CBOR_INT_32); + *(uint32_t *)item->data = value; +} + +void cbor_set_uint64(cbor_item_t *item, uint64_t value) { + CBOR_ASSERT(cbor_is_int(item)); + CBOR_ASSERT(cbor_int_get_width(item) == CBOR_INT_64); + *(uint64_t *)item->data = value; +} + +void cbor_mark_uint(cbor_item_t *item) { + CBOR_ASSERT(cbor_is_int(item)); + item->type = CBOR_TYPE_UINT; +} + +void cbor_mark_negint(cbor_item_t *item) { + CBOR_ASSERT(cbor_is_int(item)); + item->type = CBOR_TYPE_NEGINT; +} + +cbor_item_t *cbor_new_int8(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t) + 1); + _CBOR_NOTNULL(item); + *item = (cbor_item_t){.data = (unsigned char *)item + sizeof(cbor_item_t), + .refcount = 1, + .metadata = {.int_metadata = {.width = CBOR_INT_8}}, + .type = CBOR_TYPE_UINT}; + return item; +} + +cbor_item_t *cbor_new_int16(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t) + 2); + _CBOR_NOTNULL(item); + *item = (cbor_item_t){.data = (unsigned char *)item + sizeof(cbor_item_t), + .refcount = 1, + .metadata = {.int_metadata = {.width = CBOR_INT_16}}, + .type = CBOR_TYPE_UINT}; + return item; +} + +cbor_item_t *cbor_new_int32(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t) + 4); + _CBOR_NOTNULL(item); + *item = (cbor_item_t){.data = (unsigned char *)item + sizeof(cbor_item_t), + .refcount = 1, + .metadata = {.int_metadata = {.width = CBOR_INT_32}}, + .type = CBOR_TYPE_UINT}; + return item; +} + +cbor_item_t *cbor_new_int64(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t) + 8); + _CBOR_NOTNULL(item); + *item = (cbor_item_t){.data = (unsigned char *)item + sizeof(cbor_item_t), + .refcount = 1, + .metadata = {.int_metadata = {.width = CBOR_INT_64}}, + .type = CBOR_TYPE_UINT}; + return item; +} + +cbor_item_t *cbor_build_uint8(uint8_t value) { + cbor_item_t *item = cbor_new_int8(); + _CBOR_NOTNULL(item); + cbor_set_uint8(item, value); + cbor_mark_uint(item); + return item; +} + +cbor_item_t *cbor_build_uint16(uint16_t value) { + cbor_item_t *item = cbor_new_int16(); + _CBOR_NOTNULL(item); + cbor_set_uint16(item, value); + cbor_mark_uint(item); + return item; +} + +cbor_item_t *cbor_build_uint32(uint32_t value) { + cbor_item_t *item = cbor_new_int32(); + _CBOR_NOTNULL(item); + cbor_set_uint32(item, value); + cbor_mark_uint(item); + return item; +} + +cbor_item_t *cbor_build_uint64(uint64_t value) { + cbor_item_t *item = cbor_new_int64(); + _CBOR_NOTNULL(item); + cbor_set_uint64(item, value); + cbor_mark_uint(item); + return item; +} + +cbor_item_t *cbor_build_negint8(uint8_t value) { + cbor_item_t *item = cbor_new_int8(); + _CBOR_NOTNULL(item); + cbor_set_uint8(item, value); + cbor_mark_negint(item); + return item; +} + +cbor_item_t *cbor_build_negint16(uint16_t value) { + cbor_item_t *item = cbor_new_int16(); + _CBOR_NOTNULL(item); + cbor_set_uint16(item, value); + cbor_mark_negint(item); + return item; +} + +cbor_item_t *cbor_build_negint32(uint32_t value) { + cbor_item_t *item = cbor_new_int32(); + _CBOR_NOTNULL(item); + cbor_set_uint32(item, value); + cbor_mark_negint(item); + return item; +} + +cbor_item_t *cbor_build_negint64(uint64_t value) { + cbor_item_t *item = cbor_new_int64(); + _CBOR_NOTNULL(item); + cbor_set_uint64(item, value); + cbor_mark_negint(item); + return item; +} diff --git a/source/external/libcbor/cbor/ints.h b/source/external/libcbor/cbor/ints.h new file mode 100644 index 000000000..006aa428e --- /dev/null +++ b/source/external/libcbor/cbor/ints.h @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_INTS_H +#define LIBCBOR_INTS_H + +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * ============================================================================ + * Integer (uints and negints) manipulation + * ============================================================================ + */ + +/** Extracts the integer value + * + * @param item positive or negative integer + * @return the value + */ +_CBOR_NODISCARD CBOR_EXPORT uint8_t cbor_get_uint8(const cbor_item_t *item); + +/** Extracts the integer value + * + * @param item positive or negative integer + * @return the value + */ +_CBOR_NODISCARD CBOR_EXPORT uint16_t cbor_get_uint16(const cbor_item_t *item); + +/** Extracts the integer value + * + * @param item positive or negative integer + * @return the value + */ +_CBOR_NODISCARD CBOR_EXPORT uint32_t cbor_get_uint32(const cbor_item_t *item); + +/** Extracts the integer value + * + * @param item positive or negative integer + * @return the value + */ +_CBOR_NODISCARD CBOR_EXPORT uint64_t cbor_get_uint64(const cbor_item_t *item); + +/** Extracts the integer value + * + * @param item positive or negative integer + * @return the value, extended to `uint64_t` + */ +_CBOR_NODISCARD CBOR_EXPORT uint64_t cbor_get_int(const cbor_item_t *item); + +/** Assigns the integer value + * + * @param item positive or negative integer item + * @param value the value to assign. For negative integer, the logical value is + * `-value - 1` + */ +CBOR_EXPORT void cbor_set_uint8(cbor_item_t *item, uint8_t value); + +/** Assigns the integer value + * + * @param item positive or negative integer item + * @param value the value to assign. For negative integer, the logical value is + * `-value - 1` + */ +CBOR_EXPORT void cbor_set_uint16(cbor_item_t *item, uint16_t value); + +/** Assigns the integer value + * + * @param item positive or negative integer item + * @param value the value to assign. For negative integer, the logical value is + * `-value - 1` + */ +CBOR_EXPORT void cbor_set_uint32(cbor_item_t *item, uint32_t value); + +/** Assigns the integer value + * + * @param item positive or negative integer item + * @param value the value to assign. For negative integer, the logical value is + * `-value - 1` + */ +CBOR_EXPORT void cbor_set_uint64(cbor_item_t *item, uint64_t value); + +/** Queries the integer width + * + * @param item positive or negative integer item + * @return the width + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_int_width +cbor_int_get_width(const cbor_item_t *item); + +/** Marks the integer item as a positive integer + * + * The data value is not changed + * + * @param item positive or negative integer item + */ +CBOR_EXPORT void cbor_mark_uint(cbor_item_t *item); + +/** Marks the integer item as a negative integer + * + * The data value is not changed + * + * @param item positive or negative integer item + */ +CBOR_EXPORT void cbor_mark_negint(cbor_item_t *item); + +/** Allocates new integer with 1B width + * + * The width cannot be changed once allocated + * + * @return **new** positive integer or `NULL` on memory allocation failure. The + * value is not initialized + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_int8(void); + +/** Allocates new integer with 2B width + * + * The width cannot be changed once allocated + * + * @return **new** positive integer or `NULL` on memory allocation failure. The + * value is not initialized + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_int16(void); + +/** Allocates new integer with 4B width + * + * The width cannot be changed once allocated + * + * @return **new** positive integer or `NULL` on memory allocation failure. The + * value is not initialized + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_int32(void); + +/** Allocates new integer with 8B width + * + * The width cannot be changed once allocated + * + * @return **new** positive integer or `NULL` on memory allocation failure. The + * value is not initialized + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_int64(void); + +/** Constructs a new positive integer + * + * @param value the value to use + * @return **new** positive integer or `NULL` on memory allocation failure + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_uint8(uint8_t value); + +/** Constructs a new positive integer + * + * @param value the value to use + * @return **new** positive integer or `NULL` on memory allocation failure + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_uint16(uint16_t value); + +/** Constructs a new positive integer + * + * @param value the value to use + * @return **new** positive integer or `NULL` on memory allocation failure + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_uint32(uint32_t value); + +/** Constructs a new positive integer + * + * @param value the value to use + * @return **new** positive integer or `NULL` on memory allocation failure + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_uint64(uint64_t value); + +/** Constructs a new negative integer + * + * @param value the value to use + * @return **new** negative integer or `NULL` on memory allocation failure + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_negint8(uint8_t value); + +/** Constructs a new negative integer + * + * @param value the value to use + * @return **new** negative integer or `NULL` on memory allocation failure + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_negint16(uint16_t value); + +/** Constructs a new negative integer + * + * @param value the value to use + * @return **new** negative integer or `NULL` on memory allocation failure + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_negint32(uint32_t value); + +/** Constructs a new negative integer + * + * @param value the value to use + * @return **new** negative integer or `NULL` on memory allocation failure + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_negint64(uint64_t value); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_INTS_H diff --git a/source/external/libcbor/cbor/maps.c b/source/external/libcbor/cbor/maps.c new file mode 100644 index 000000000..8a3bd6075 --- /dev/null +++ b/source/external/libcbor/cbor/maps.c @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "maps.h" +#include "internal/memory_utils.h" + +size_t cbor_map_size(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_map(item)); + return item->metadata.map_metadata.end_ptr; +} + +size_t cbor_map_allocated(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_map(item)); + return item->metadata.map_metadata.allocated; +} + +cbor_item_t *cbor_new_definite_map(size_t size) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t)); + _CBOR_NOTNULL(item); + + *item = (cbor_item_t){ + .refcount = 1, + .type = CBOR_TYPE_MAP, + .metadata = {.map_metadata = {.allocated = size, + .type = _CBOR_METADATA_DEFINITE, + .end_ptr = 0}}, + .data = _cbor_alloc_multiple(sizeof(struct cbor_pair), size)}; + _CBOR_DEPENDENT_NOTNULL(item, item->data); + + return item; +} + +cbor_item_t *cbor_new_indefinite_map(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t)); + _CBOR_NOTNULL(item); + + *item = (cbor_item_t){ + .refcount = 1, + .type = CBOR_TYPE_MAP, + .metadata = {.map_metadata = {.allocated = 0, + .type = _CBOR_METADATA_INDEFINITE, + .end_ptr = 0}}, + .data = NULL}; + + return item; +} + +bool _cbor_map_add_key(cbor_item_t *item, cbor_item_t *key) { + CBOR_ASSERT(cbor_isa_map(item)); + struct _cbor_map_metadata *metadata = + (struct _cbor_map_metadata *)&item->metadata; + if (cbor_map_is_definite(item)) { + struct cbor_pair *data = cbor_map_handle(item); + if (metadata->end_ptr >= metadata->allocated) { + /* Don't realloc definite preallocated map */ + return false; + } + + data[metadata->end_ptr].key = key; + data[metadata->end_ptr++].value = NULL; + } else { + if (metadata->end_ptr >= metadata->allocated) { + /* Exponential realloc */ + // Check for overflows first + if (!_cbor_safe_to_multiply(CBOR_BUFFER_GROWTH, metadata->allocated)) { + return false; + } + + size_t new_allocation = metadata->allocated == 0 + ? 1 + : CBOR_BUFFER_GROWTH * metadata->allocated; + + unsigned char *new_data = _cbor_realloc_multiple( + item->data, sizeof(struct cbor_pair), new_allocation); + + if (new_data == NULL) { + return false; + } + + item->data = new_data; + metadata->allocated = new_allocation; + } + struct cbor_pair *data = cbor_map_handle(item); + data[metadata->end_ptr].key = key; + data[metadata->end_ptr++].value = NULL; + } + cbor_incref(key); + return true; +} + +bool _cbor_map_add_value(cbor_item_t *item, cbor_item_t *value) { + CBOR_ASSERT(cbor_isa_map(item)); + cbor_incref(value); + cbor_map_handle(item)[ + /* Move one back since we are assuming _add_key (which increased the ptr) + * was the previous operation on this object */ + item->metadata.map_metadata.end_ptr - 1] + .value = value; + return true; +} + +// TODO: Add a more convenient API like add(item, key, val) +bool cbor_map_add(cbor_item_t *item, struct cbor_pair pair) { + CBOR_ASSERT(cbor_isa_map(item)); + if (!_cbor_map_add_key(item, pair.key)) return false; + return _cbor_map_add_value(item, pair.value); +} + +bool cbor_map_is_definite(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_map(item)); + return item->metadata.map_metadata.type == _CBOR_METADATA_DEFINITE; +} + +bool cbor_map_is_indefinite(const cbor_item_t *item) { + return !cbor_map_is_definite(item); +} + +struct cbor_pair *cbor_map_handle(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_map(item)); + return (struct cbor_pair *)item->data; +} diff --git a/source/external/libcbor/cbor/maps.h b/source/external/libcbor/cbor/maps.h new file mode 100644 index 000000000..5c05b542b --- /dev/null +++ b/source/external/libcbor/cbor/maps.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_MAPS_H +#define LIBCBOR_MAPS_H + +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * ============================================================================ + * Map manipulation + * ============================================================================ + */ + +/** Get the number of pairs + * + * @param item A map + * @return The number of pairs + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_map_size(const cbor_item_t *item); + +/** Get the size of the allocated storage + * + * @param item A map + * @return Allocated storage size (as the number of #cbor_pair items) + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_map_allocated(const cbor_item_t *item); + +/** Create a new definite map + * + * @param size The number of slots to preallocate + * @return Reference to the new map item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_definite_map(size_t size); + +/** Create a new indefinite map + * + * @return Reference to the new map item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_indefinite_map(void); + +/** Add a pair to the map + * + * For definite maps, items can only be added to the preallocated space. For + * indefinite maps, the storage will be expanded as needed + * + * @param item A map + * @param pair The key-value pair to add. Reference count of the #cbor_pair.key + * and #cbor_pair.value will be increased by one. + * @return `true` on success, `false` if memory allocation failed (indefinite + * maps) or the preallocated storage is full (definite maps) + */ +_CBOR_NODISCARD CBOR_EXPORT bool cbor_map_add(cbor_item_t *item, + struct cbor_pair pair); + +/** Add a key to the map + * + * Sets the value to `NULL`. Internal API. + * + * @param item A map + * @param key The key, Its reference count will be be increased by one. + * @return `true` on success, `false` if either reallocation failed or the + * preallocated storage is full + */ +_CBOR_NODISCARD CBOR_EXPORT bool _cbor_map_add_key(cbor_item_t *item, + cbor_item_t *key); + +/** Add a value to the map + * + * Assumes that #_cbor_map_add_key has been called. Internal API. + * + * @param item A map + * @param value The value. Its reference count will be be increased by one. + * @return `true` on success, `false` if either reallocation failed or the + * preallocated storage is full + */ +_CBOR_NODISCARD CBOR_EXPORT bool _cbor_map_add_value(cbor_item_t *item, + cbor_item_t *value); + +/** Is this map definite? + * + * @param item A map + * @return Is this map definite? + */ +_CBOR_NODISCARD CBOR_EXPORT bool cbor_map_is_definite(const cbor_item_t *item); + +/** Is this map indefinite? + * + * @param item A map + * @return Is this map indefinite? + */ +_CBOR_NODISCARD CBOR_EXPORT bool cbor_map_is_indefinite( + const cbor_item_t *item); + +/** Get the pairs storage + * + * @param item A map + * @return Array of #cbor_map_size pairs. Manipulation is possible as long as + * references remain valid. + */ +_CBOR_NODISCARD CBOR_EXPORT struct cbor_pair *cbor_map_handle( + const cbor_item_t *item); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_MAPS_H diff --git a/source/external/libcbor/cbor/serialization.c b/source/external/libcbor/cbor/serialization.c new file mode 100644 index 000000000..40f4c531d --- /dev/null +++ b/source/external/libcbor/cbor/serialization.c @@ -0,0 +1,368 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "serialization.h" +#include +#include "cbor/arrays.h" +#include "cbor/bytestrings.h" +#include "cbor/floats_ctrls.h" +#include "cbor/ints.h" +#include "cbor/maps.h" +#include "cbor/strings.h" +#include "cbor/tags.h" +#include "encoding.h" +#include "internal/memory_utils.h" + +size_t cbor_serialize(const cbor_item_t *item, unsigned char *buffer, + size_t buffer_size) { + // cppcheck-suppress missingReturn + switch (cbor_typeof(item)) { + case CBOR_TYPE_UINT: + return cbor_serialize_uint(item, buffer, buffer_size); + case CBOR_TYPE_NEGINT: + return cbor_serialize_negint(item, buffer, buffer_size); + case CBOR_TYPE_BYTESTRING: + return cbor_serialize_bytestring(item, buffer, buffer_size); + case CBOR_TYPE_STRING: + return cbor_serialize_string(item, buffer, buffer_size); + case CBOR_TYPE_ARRAY: + return cbor_serialize_array(item, buffer, buffer_size); + case CBOR_TYPE_MAP: + return cbor_serialize_map(item, buffer, buffer_size); + case CBOR_TYPE_TAG: + return cbor_serialize_tag(item, buffer, buffer_size); + case CBOR_TYPE_FLOAT_CTRL: + return cbor_serialize_float_ctrl(item, buffer, buffer_size); + } +} + +/** Largest integer that can be encoded as embedded in the item leading byte. */ +const uint64_t kMaxEmbeddedInt = 23; + +/** How many bytes will a tag for a nested item of a given `size` take when + * encoded.*/ +size_t _cbor_encoded_header_size(uint64_t size) { + if (size <= kMaxEmbeddedInt) + return 1; + else if (size <= UINT8_MAX) + return 2; + else if (size <= UINT16_MAX) + return 3; + else if (size <= UINT32_MAX) + return 5; + else + return 9; +} + +size_t cbor_serialized_size(const cbor_item_t *item) { + // cppcheck-suppress missingReturn + switch (cbor_typeof(item)) { + case CBOR_TYPE_UINT: + case CBOR_TYPE_NEGINT: + switch (cbor_int_get_width(item)) { + case CBOR_INT_8: + if (cbor_get_uint8(item) <= kMaxEmbeddedInt) return 1; + return 2; + case CBOR_INT_16: + return 3; + case CBOR_INT_32: + return 5; + case CBOR_INT_64: + return 9; + } + // Note: We do not _cbor_safe_signaling_add zero-length definite strings, + // they would cause zeroes to propagate. All other items are at least one + // byte. + case CBOR_TYPE_BYTESTRING: { + if (cbor_bytestring_is_definite(item)) { + size_t header_size = + _cbor_encoded_header_size(cbor_bytestring_length(item)); + if (cbor_bytestring_length(item) == 0) return header_size; + return _cbor_safe_signaling_add(header_size, + cbor_bytestring_length(item)); + } + size_t indef_bytestring_size = 2; // Leading byte + break + cbor_item_t **chunks = cbor_bytestring_chunks_handle(item); + for (size_t i = 0; i < cbor_bytestring_chunk_count(item); i++) { + indef_bytestring_size = _cbor_safe_signaling_add( + indef_bytestring_size, cbor_serialized_size(chunks[i])); + } + return indef_bytestring_size; + } + case CBOR_TYPE_STRING: { + if (cbor_string_is_definite(item)) { + size_t header_size = + _cbor_encoded_header_size(cbor_string_length(item)); + if (cbor_string_length(item) == 0) return header_size; + return _cbor_safe_signaling_add(header_size, cbor_string_length(item)); + } + size_t indef_string_size = 2; // Leading byte + break + cbor_item_t **chunks = cbor_string_chunks_handle(item); + for (size_t i = 0; i < cbor_string_chunk_count(item); i++) { + indef_string_size = _cbor_safe_signaling_add( + indef_string_size, cbor_serialized_size(chunks[i])); + } + return indef_string_size; + } + case CBOR_TYPE_ARRAY: { + size_t array_size = cbor_array_is_definite(item) + ? _cbor_encoded_header_size(cbor_array_size(item)) + : 2; // Leading byte + break + cbor_item_t **items = cbor_array_handle(item); + for (size_t i = 0; i < cbor_array_size(item); i++) { + array_size = _cbor_safe_signaling_add(array_size, + cbor_serialized_size(items[i])); + } + return array_size; + } + case CBOR_TYPE_MAP: { + size_t map_size = cbor_map_is_definite(item) + ? _cbor_encoded_header_size(cbor_map_size(item)) + : 2; // Leading byte + break + struct cbor_pair *items = cbor_map_handle(item); + for (size_t i = 0; i < cbor_map_size(item); i++) { + map_size = _cbor_safe_signaling_add( + map_size, + _cbor_safe_signaling_add(cbor_serialized_size(items[i].key), + cbor_serialized_size(items[i].value))); + } + return map_size; + } + case CBOR_TYPE_TAG: { + return _cbor_safe_signaling_add( + _cbor_encoded_header_size(cbor_tag_value(item)), + cbor_serialized_size(cbor_move(cbor_tag_item(item)))); + } + case CBOR_TYPE_FLOAT_CTRL: + switch (cbor_float_get_width(item)) { + case CBOR_FLOAT_0: + return _cbor_encoded_header_size(cbor_ctrl_value(item)); + case CBOR_FLOAT_16: + return 3; + case CBOR_FLOAT_32: + return 5; + case CBOR_FLOAT_64: + return 9; + } + } +} + +size_t cbor_serialize_alloc(const cbor_item_t *item, unsigned char **buffer, + size_t *buffer_size) { + *buffer = NULL; + size_t serialized_size = cbor_serialized_size(item); + if (serialized_size == 0) { + if (buffer_size != NULL) *buffer_size = 0; + return 0; + } + *buffer = _cbor_malloc(serialized_size); + if (*buffer == NULL) { + if (buffer_size != NULL) *buffer_size = 0; + return 0; + } + + size_t written = cbor_serialize(item, *buffer, serialized_size); + CBOR_ASSERT(written == serialized_size); + if (buffer_size != NULL) *buffer_size = serialized_size; + return written; +} + +size_t cbor_serialize_uint(const cbor_item_t *item, unsigned char *buffer, + size_t buffer_size) { + CBOR_ASSERT(cbor_isa_uint(item)); + // cppcheck-suppress missingReturn + switch (cbor_int_get_width(item)) { + case CBOR_INT_8: + return cbor_encode_uint8(cbor_get_uint8(item), buffer, buffer_size); + case CBOR_INT_16: + return cbor_encode_uint16(cbor_get_uint16(item), buffer, buffer_size); + case CBOR_INT_32: + return cbor_encode_uint32(cbor_get_uint32(item), buffer, buffer_size); + case CBOR_INT_64: + return cbor_encode_uint64(cbor_get_uint64(item), buffer, buffer_size); + } +} + +size_t cbor_serialize_negint(const cbor_item_t *item, unsigned char *buffer, + size_t buffer_size) { + CBOR_ASSERT(cbor_isa_negint(item)); + // cppcheck-suppress missingReturn + switch (cbor_int_get_width(item)) { + case CBOR_INT_8: + return cbor_encode_negint8(cbor_get_uint8(item), buffer, buffer_size); + case CBOR_INT_16: + return cbor_encode_negint16(cbor_get_uint16(item), buffer, buffer_size); + case CBOR_INT_32: + return cbor_encode_negint32(cbor_get_uint32(item), buffer, buffer_size); + case CBOR_INT_64: + return cbor_encode_negint64(cbor_get_uint64(item), buffer, buffer_size); + } +} + +size_t cbor_serialize_bytestring(const cbor_item_t *item, unsigned char *buffer, + size_t buffer_size) { + CBOR_ASSERT(cbor_isa_bytestring(item)); + if (cbor_bytestring_is_definite(item)) { + size_t length = cbor_bytestring_length(item); + size_t written = cbor_encode_bytestring_start(length, buffer, buffer_size); + if (written > 0 && (buffer_size - written >= length)) { + memcpy(buffer + written, cbor_bytestring_handle(item), length); + return written + length; + } + return 0; + } else { + CBOR_ASSERT(cbor_bytestring_is_indefinite(item)); + size_t chunk_count = cbor_bytestring_chunk_count(item); + size_t written = cbor_encode_indef_bytestring_start(buffer, buffer_size); + if (written == 0) return 0; + + cbor_item_t **chunks = cbor_bytestring_chunks_handle(item); + for (size_t i = 0; i < chunk_count; i++) { + size_t chunk_written = cbor_serialize_bytestring( + chunks[i], buffer + written, buffer_size - written); + if (chunk_written == 0) return 0; + written += chunk_written; + } + + size_t break_written = + cbor_encode_break(buffer + written, buffer_size - written); + if (break_written == 0) return 0; + return written + break_written; + } +} + +size_t cbor_serialize_string(const cbor_item_t *item, unsigned char *buffer, + size_t buffer_size) { + CBOR_ASSERT(cbor_isa_string(item)); + if (cbor_string_is_definite(item)) { + size_t length = cbor_string_length(item); + size_t written = cbor_encode_string_start(length, buffer, buffer_size); + if (written && (buffer_size - written >= length)) { + memcpy(buffer + written, cbor_string_handle(item), length); + return written + length; + } + return 0; + } else { + CBOR_ASSERT(cbor_string_is_indefinite(item)); + size_t chunk_count = cbor_string_chunk_count(item); + size_t written = cbor_encode_indef_string_start(buffer, buffer_size); + if (written == 0) return 0; + + cbor_item_t **chunks = cbor_string_chunks_handle(item); + for (size_t i = 0; i < chunk_count; i++) { + size_t chunk_written = cbor_serialize_string(chunks[i], buffer + written, + buffer_size - written); + if (chunk_written == 0) return 0; + written += chunk_written; + } + + size_t break_written = + cbor_encode_break(buffer + written, buffer_size - written); + if (break_written == 0) return 0; + return written + break_written; + } +} + +size_t cbor_serialize_array(const cbor_item_t *item, unsigned char *buffer, + size_t buffer_size) { + CBOR_ASSERT(cbor_isa_array(item)); + size_t size = cbor_array_size(item), written = 0; + cbor_item_t **handle = cbor_array_handle(item); + if (cbor_array_is_definite(item)) { + written = cbor_encode_array_start(size, buffer, buffer_size); + } else { + CBOR_ASSERT(cbor_array_is_indefinite(item)); + written = cbor_encode_indef_array_start(buffer, buffer_size); + } + if (written == 0) return 0; + + for (size_t i = 0; i < size; i++) { + size_t item_written = + cbor_serialize(*(handle++), buffer + written, buffer_size - written); + if (item_written == 0) return 0; + written += item_written; + } + + if (cbor_array_is_definite(item)) { + return written; + } else { + CBOR_ASSERT(cbor_array_is_indefinite(item)); + size_t break_written = + cbor_encode_break(buffer + written, buffer_size - written); + if (break_written == 0) return 0; + return written + break_written; + } +} + +size_t cbor_serialize_map(const cbor_item_t *item, unsigned char *buffer, + size_t buffer_size) { + CBOR_ASSERT(cbor_isa_map(item)); + size_t size = cbor_map_size(item), written = 0; + struct cbor_pair *handle = cbor_map_handle(item); + + if (cbor_map_is_definite(item)) { + written = cbor_encode_map_start(size, buffer, buffer_size); + } else { + CBOR_ASSERT(cbor_map_is_indefinite(item)); + written = cbor_encode_indef_map_start(buffer, buffer_size); + } + if (written == 0) return 0; + + for (size_t i = 0; i < size; i++) { + size_t item_written = + cbor_serialize(handle->key, buffer + written, buffer_size - written); + if (item_written == 0) { + return 0; + } + written += item_written; + item_written = cbor_serialize((handle++)->value, buffer + written, + buffer_size - written); + if (item_written == 0) return 0; + written += item_written; + } + + if (cbor_map_is_definite(item)) { + return written; + } else { + CBOR_ASSERT(cbor_map_is_indefinite(item)); + size_t break_written = + cbor_encode_break(buffer + written, buffer_size - written); + if (break_written == 0) return 0; + return written + break_written; + } +} + +size_t cbor_serialize_tag(const cbor_item_t *item, unsigned char *buffer, + size_t buffer_size) { + CBOR_ASSERT(cbor_isa_tag(item)); + size_t written = cbor_encode_tag(cbor_tag_value(item), buffer, buffer_size); + if (written == 0) return 0; + + size_t item_written = cbor_serialize(cbor_move(cbor_tag_item(item)), + buffer + written, buffer_size - written); + if (item_written == 0) return 0; + return written + item_written; +} + +size_t cbor_serialize_float_ctrl(const cbor_item_t *item, unsigned char *buffer, + size_t buffer_size) { + CBOR_ASSERT(cbor_isa_float_ctrl(item)); + // cppcheck-suppress missingReturn + switch (cbor_float_get_width(item)) { + case CBOR_FLOAT_0: + /* CTRL - special treatment */ + return cbor_encode_ctrl(cbor_ctrl_value(item), buffer, buffer_size); + case CBOR_FLOAT_16: + return cbor_encode_half(cbor_float_get_float2(item), buffer, buffer_size); + case CBOR_FLOAT_32: + return cbor_encode_single(cbor_float_get_float4(item), buffer, + buffer_size); + case CBOR_FLOAT_64: + return cbor_encode_double(cbor_float_get_float8(item), buffer, + buffer_size); + } +} diff --git a/source/external/libcbor/cbor/serialization.h b/source/external/libcbor/cbor/serialization.h new file mode 100644 index 000000000..228ae75d6 --- /dev/null +++ b/source/external/libcbor/cbor/serialization.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_SERIALIZATION_H +#define LIBCBOR_SERIALIZATION_H + +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * ============================================================================ + * High level encoding + * ============================================================================ + */ + +/** Serialize the given item + * + * @param item A data item + * @param buffer Buffer to serialize to + * @param buffer_size Size of the \p buffer + * @return Length of the result. 0 on failure. + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_serialize(const cbor_item_t *item, + cbor_mutable_data buffer, + size_t buffer_size); + +/** Compute the length (in bytes) of the item when serialized using + * `cbor_serialize`. + * + * Time complexity is proportional to the number of nested items. + * + * @param item A data item + * @return Length (>= 1) of the item when serialized. 0 if the length overflows + * `size_t`. + */ +_CBOR_NODISCARD CBOR_EXPORT size_t +cbor_serialized_size(const cbor_item_t *item); + +/** Serialize the given item, allocating buffers as needed + * + * Since libcbor v0.10, the return value is always the same as `buffer_size` (if + * provided, see https://github.com/PJK/libcbor/pull/251/). New clients should + * ignore the return value. + * + * \rst + * .. warning:: It is the caller's responsibility to free the buffer using an + * appropriate ``free`` implementation. + * \endrst + * + * @param item A data item + * @param[out] buffer Buffer containing the result + * @param[out] buffer_size Size of the \p buffer, or 0 on memory allocation + * failure. + * @return Length of the result in bytes + * @return 0 on memory allocation failure, in which case \p buffer is `NULL`. + */ +CBOR_EXPORT size_t cbor_serialize_alloc(const cbor_item_t *item, + unsigned char **buffer, + size_t *buffer_size); + +/** Serialize an uint + * + * @param item A uint + * @param[out] buffer Buffer to serialize to + * @param buffer_size Size of the \p buffer + * @return Length of the result + * @return 0 if the \p buffer_size doesn't fit the result + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_serialize_uint(const cbor_item_t *item, + cbor_mutable_data buffer, + size_t buffer_size); + +/** Serialize a negint + * + * @param item A negint + * @param[out] buffer Buffer to serialize to + * @param buffer_size Size of the \p buffer + * @return Length of the result + * @return 0 if the \p buffer_size doesn't fit the result + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_serialize_negint( + const cbor_item_t *item, cbor_mutable_data buffer, size_t buffer_size); + +/** Serialize a bytestring + * + * @param item A bytestring + * @param[out] buffer Buffer to serialize to + * @param buffer_size Size of the \p buffer + * @return Length of the result + * @return 0 if the \p buffer_size doesn't fit the result. The \p buffer may + * still be modified + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_serialize_bytestring( + const cbor_item_t *item, cbor_mutable_data buffer, size_t buffer_size); + +/** Serialize a string + * + * @param item A string + * @param[out] buffer Buffer to serialize to + * @param buffer_size Size of the \p buffer + * @return Length of the result + * @return 0 if the \p buffer_size doesn't fit the result. The \p buffer may + * still be modified + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_serialize_string( + const cbor_item_t *item, cbor_mutable_data buffer, size_t buffer_size); +/** Serialize an array + * + * @param item An array + * @param[out] buffer Buffer to serialize to + * @param buffer_size Size of the \p buffer + * @return Length of the result + * @return 0 if the \p buffer_size doesn't fit the result. The \p buffer may + * still be modified + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_serialize_array( + const cbor_item_t *item, cbor_mutable_data buffer, size_t buffer_size); + +/** Serialize a map + * + * @param item A map + * @param[out] buffer Buffer to serialize to + * @param buffer_size Size of the \p buffer + * @return Length of the result + * @return 0 if the \p buffer_size doesn't fit the result. The \p buffer may + * still be modified + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_serialize_map(const cbor_item_t *item, + cbor_mutable_data buffer, + size_t buffer_size); + +/** Serialize a tag + * + * @param item A tag + * @param[out] buffer Buffer to serialize to + * @param buffer_size Size of the \p buffer + * @return Length of the result + * @return 0 if the \p buffer_size doesn't fit the result. The \p buffer may + * still be modified + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_serialize_tag(const cbor_item_t *item, + cbor_mutable_data buffer, + size_t buffer_size); + +/** Serialize a + * + * @param item A float or ctrl + * @param[out] buffer Buffer to serialize to + * @param buffer_size Size of the \p buffer + * @return Length of the result + * @return 0 if the \p buffer_size doesn't fit the result + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_serialize_float_ctrl( + const cbor_item_t *item, cbor_mutable_data buffer, size_t buffer_size); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_SERIALIZATION_H diff --git a/source/external/libcbor/cbor/streaming.c b/source/external/libcbor/cbor/streaming.c new file mode 100644 index 000000000..4b3770114 --- /dev/null +++ b/source/external/libcbor/cbor/streaming.c @@ -0,0 +1,600 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "streaming.h" +#include "internal/loaders.h" + +static bool claim_bytes(size_t required, size_t provided, + struct cbor_decoder_result *result) { + if (required > (provided - result->read)) { + result->required = required + result->read; + result->read = 0; + result->status = CBOR_DECODER_NEDATA; + return false; + } else { + result->read += required; + result->required = 0; + return true; + } +} + +// Use implicit capture as an exception to avoid the super long parameter list +#define CLAIM_BYTES_AND_INVOKE(callback_name, length, source_extra_offset) \ + do { \ + if (claim_bytes(length, source_size, &result)) { \ + callbacks->callback_name(context, source + 1 + source_extra_offset, \ + length); \ + } \ + } while (0) + +#define READ_CLAIM_INVOKE(callback_name, length_reader, length_bytes) \ + do { \ + if (claim_bytes(length_bytes, source_size, &result)) { \ + uint64_t length = length_reader(source + 1); \ + CLAIM_BYTES_AND_INVOKE(callback_name, length, length_bytes); \ + } \ + return result; \ + } while (0) + +struct cbor_decoder_result cbor_stream_decode( + cbor_data source, size_t source_size, + const struct cbor_callbacks *callbacks, void *context) { + // Attempt to claim the initial MTB byte + struct cbor_decoder_result result = {.status = CBOR_DECODER_FINISHED}; + if (!claim_bytes(1, source_size, &result)) { + return result; + } + + switch (*source) { + case 0x00: /* Fallthrough */ + case 0x01: /* Fallthrough */ + case 0x02: /* Fallthrough */ + case 0x03: /* Fallthrough */ + case 0x04: /* Fallthrough */ + case 0x05: /* Fallthrough */ + case 0x06: /* Fallthrough */ + case 0x07: /* Fallthrough */ + case 0x08: /* Fallthrough */ + case 0x09: /* Fallthrough */ + case 0x0A: /* Fallthrough */ + case 0x0B: /* Fallthrough */ + case 0x0C: /* Fallthrough */ + case 0x0D: /* Fallthrough */ + case 0x0E: /* Fallthrough */ + case 0x0F: /* Fallthrough */ + case 0x10: /* Fallthrough */ + case 0x11: /* Fallthrough */ + case 0x12: /* Fallthrough */ + case 0x13: /* Fallthrough */ + case 0x14: /* Fallthrough */ + case 0x15: /* Fallthrough */ + case 0x16: /* Fallthrough */ + case 0x17: + /* Embedded one byte unsigned integer */ + { + callbacks->uint8(context, _cbor_load_uint8(source)); + return result; + } + case 0x18: + /* One byte unsigned integer */ + { + if (claim_bytes(1, source_size, &result)) { + callbacks->uint8(context, _cbor_load_uint8(source + 1)); + } + return result; + } + case 0x19: + /* Two bytes unsigned integer */ + { + if (claim_bytes(2, source_size, &result)) { + callbacks->uint16(context, _cbor_load_uint16(source + 1)); + } + return result; + } + case 0x1A: + /* Four bytes unsigned integer */ + { + if (claim_bytes(4, source_size, &result)) { + callbacks->uint32(context, _cbor_load_uint32(source + 1)); + } + return result; + } + case 0x1B: + /* Eight bytes unsigned integer */ + { + if (claim_bytes(8, source_size, &result)) { + callbacks->uint64(context, _cbor_load_uint64(source + 1)); + } + return result; + } + case 0x1C: /* Fallthrough */ + case 0x1D: /* Fallthrough */ + case 0x1E: /* Fallthrough */ + case 0x1F: + /* Reserved */ + { return (struct cbor_decoder_result){.status = CBOR_DECODER_ERROR}; } + case 0x20: /* Fallthrough */ + case 0x21: /* Fallthrough */ + case 0x22: /* Fallthrough */ + case 0x23: /* Fallthrough */ + case 0x24: /* Fallthrough */ + case 0x25: /* Fallthrough */ + case 0x26: /* Fallthrough */ + case 0x27: /* Fallthrough */ + case 0x28: /* Fallthrough */ + case 0x29: /* Fallthrough */ + case 0x2A: /* Fallthrough */ + case 0x2B: /* Fallthrough */ + case 0x2C: /* Fallthrough */ + case 0x2D: /* Fallthrough */ + case 0x2E: /* Fallthrough */ + case 0x2F: /* Fallthrough */ + case 0x30: /* Fallthrough */ + case 0x31: /* Fallthrough */ + case 0x32: /* Fallthrough */ + case 0x33: /* Fallthrough */ + case 0x34: /* Fallthrough */ + case 0x35: /* Fallthrough */ + case 0x36: /* Fallthrough */ + case 0x37: + /* Embedded one byte negative integer */ + { + callbacks->negint8(context, + _cbor_load_uint8(source) - 0x20); /* 0x20 offset */ + return result; + } + case 0x38: + /* One byte negative integer */ + { + if (claim_bytes(1, source_size, &result)) { + callbacks->negint8(context, _cbor_load_uint8(source + 1)); + } + return result; + } + case 0x39: + /* Two bytes negative integer */ + { + if (claim_bytes(2, source_size, &result)) { + callbacks->negint16(context, _cbor_load_uint16(source + 1)); + } + return result; + } + case 0x3A: + /* Four bytes negative integer */ + { + if (claim_bytes(4, source_size, &result)) { + callbacks->negint32(context, _cbor_load_uint32(source + 1)); + } + return result; + } + case 0x3B: + /* Eight bytes negative integer */ + { + if (claim_bytes(8, source_size, &result)) { + callbacks->negint64(context, _cbor_load_uint64(source + 1)); + } + return result; + } + case 0x3C: /* Fallthrough */ + case 0x3D: /* Fallthrough */ + case 0x3E: /* Fallthrough */ + case 0x3F: + /* Reserved */ + { return (struct cbor_decoder_result){.status = CBOR_DECODER_ERROR}; } + case 0x40: /* Fallthrough */ + case 0x41: /* Fallthrough */ + case 0x42: /* Fallthrough */ + case 0x43: /* Fallthrough */ + case 0x44: /* Fallthrough */ + case 0x45: /* Fallthrough */ + case 0x46: /* Fallthrough */ + case 0x47: /* Fallthrough */ + case 0x48: /* Fallthrough */ + case 0x49: /* Fallthrough */ + case 0x4A: /* Fallthrough */ + case 0x4B: /* Fallthrough */ + case 0x4C: /* Fallthrough */ + case 0x4D: /* Fallthrough */ + case 0x4E: /* Fallthrough */ + case 0x4F: /* Fallthrough */ + case 0x50: /* Fallthrough */ + case 0x51: /* Fallthrough */ + case 0x52: /* Fallthrough */ + case 0x53: /* Fallthrough */ + case 0x54: /* Fallthrough */ + case 0x55: /* Fallthrough */ + case 0x56: /* Fallthrough */ + case 0x57: + /* Embedded length byte string */ + { + uint64_t length = _cbor_load_uint8(source) - 0x40; /* 0x40 offset */ + CLAIM_BYTES_AND_INVOKE(byte_string, length, 0); + return result; + } + case 0x58: + /* One byte length byte string */ + READ_CLAIM_INVOKE(byte_string, _cbor_load_uint8, 1); + case 0x59: + /* Two bytes length byte string */ + READ_CLAIM_INVOKE(byte_string, _cbor_load_uint16, 2); + case 0x5A: + /* Four bytes length byte string */ + READ_CLAIM_INVOKE(byte_string, _cbor_load_uint32, 4); + case 0x5B: + /* Eight bytes length byte string */ + READ_CLAIM_INVOKE(byte_string, _cbor_load_uint64, 8); + case 0x5C: /* Fallthrough */ + case 0x5D: /* Fallthrough */ + case 0x5E: + /* Reserved */ + { return (struct cbor_decoder_result){.status = CBOR_DECODER_ERROR}; } + case 0x5F: + /* Indefinite byte string */ + { + callbacks->byte_string_start(context); + return result; + } + case 0x60: /* Fallthrough */ + case 0x61: /* Fallthrough */ + case 0x62: /* Fallthrough */ + case 0x63: /* Fallthrough */ + case 0x64: /* Fallthrough */ + case 0x65: /* Fallthrough */ + case 0x66: /* Fallthrough */ + case 0x67: /* Fallthrough */ + case 0x68: /* Fallthrough */ + case 0x69: /* Fallthrough */ + case 0x6A: /* Fallthrough */ + case 0x6B: /* Fallthrough */ + case 0x6C: /* Fallthrough */ + case 0x6D: /* Fallthrough */ + case 0x6E: /* Fallthrough */ + case 0x6F: /* Fallthrough */ + case 0x70: /* Fallthrough */ + case 0x71: /* Fallthrough */ + case 0x72: /* Fallthrough */ + case 0x73: /* Fallthrough */ + case 0x74: /* Fallthrough */ + case 0x75: /* Fallthrough */ + case 0x76: /* Fallthrough */ + case 0x77: + /* Embedded one byte length string */ + { + uint64_t length = _cbor_load_uint8(source) - 0x60; /* 0x60 offset */ + CLAIM_BYTES_AND_INVOKE(string, length, 0); + return result; + } + case 0x78: + /* One byte length string */ + READ_CLAIM_INVOKE(string, _cbor_load_uint8, 1); + case 0x79: + /* Two bytes length string */ + READ_CLAIM_INVOKE(string, _cbor_load_uint16, 2); + case 0x7A: + /* Four bytes length string */ + READ_CLAIM_INVOKE(string, _cbor_load_uint32, 4); + case 0x7B: + /* Eight bytes length string */ + READ_CLAIM_INVOKE(string, _cbor_load_uint64, 8); + case 0x7C: /* Fallthrough */ + case 0x7D: /* Fallthrough */ + case 0x7E: + /* Reserved */ + { return (struct cbor_decoder_result){.status = CBOR_DECODER_ERROR}; } + case 0x7F: + /* Indefinite length string */ + { + callbacks->string_start(context); + return result; + } + case 0x80: /* Fallthrough */ + case 0x81: /* Fallthrough */ + case 0x82: /* Fallthrough */ + case 0x83: /* Fallthrough */ + case 0x84: /* Fallthrough */ + case 0x85: /* Fallthrough */ + case 0x86: /* Fallthrough */ + case 0x87: /* Fallthrough */ + case 0x88: /* Fallthrough */ + case 0x89: /* Fallthrough */ + case 0x8A: /* Fallthrough */ + case 0x8B: /* Fallthrough */ + case 0x8C: /* Fallthrough */ + case 0x8D: /* Fallthrough */ + case 0x8E: /* Fallthrough */ + case 0x8F: /* Fallthrough */ + case 0x90: /* Fallthrough */ + case 0x91: /* Fallthrough */ + case 0x92: /* Fallthrough */ + case 0x93: /* Fallthrough */ + case 0x94: /* Fallthrough */ + case 0x95: /* Fallthrough */ + case 0x96: /* Fallthrough */ + case 0x97: + /* Embedded one byte length array */ + { + callbacks->array_start( + context, _cbor_load_uint8(source) - 0x80); /* 0x40 offset */ + return result; + } + case 0x98: + /* One byte length array */ + { + if (claim_bytes(1, source_size, &result)) { + callbacks->array_start(context, _cbor_load_uint8(source + 1)); + } + return result; + } + case 0x99: + /* Two bytes length array */ + { + if (claim_bytes(2, source_size, &result)) { + callbacks->array_start(context, _cbor_load_uint16(source + 1)); + } + return result; + } + case 0x9A: + /* Four bytes length array */ + { + if (claim_bytes(4, source_size, &result)) { + callbacks->array_start(context, _cbor_load_uint32(source + 1)); + } + return result; + } + case 0x9B: + /* Eight bytes length array */ + { + if (claim_bytes(8, source_size, &result)) { + callbacks->array_start(context, _cbor_load_uint64(source + 1)); + } + return result; + } + case 0x9C: /* Fallthrough */ + case 0x9D: /* Fallthrough */ + case 0x9E: + /* Reserved */ + { return (struct cbor_decoder_result){.status = CBOR_DECODER_ERROR}; } + case 0x9F: + /* Indefinite length array */ + { + callbacks->indef_array_start(context); + return result; + } + case 0xA0: /* Fallthrough */ + case 0xA1: /* Fallthrough */ + case 0xA2: /* Fallthrough */ + case 0xA3: /* Fallthrough */ + case 0xA4: /* Fallthrough */ + case 0xA5: /* Fallthrough */ + case 0xA6: /* Fallthrough */ + case 0xA7: /* Fallthrough */ + case 0xA8: /* Fallthrough */ + case 0xA9: /* Fallthrough */ + case 0xAA: /* Fallthrough */ + case 0xAB: /* Fallthrough */ + case 0xAC: /* Fallthrough */ + case 0xAD: /* Fallthrough */ + case 0xAE: /* Fallthrough */ + case 0xAF: /* Fallthrough */ + case 0xB0: /* Fallthrough */ + case 0xB1: /* Fallthrough */ + case 0xB2: /* Fallthrough */ + case 0xB3: /* Fallthrough */ + case 0xB4: /* Fallthrough */ + case 0xB5: /* Fallthrough */ + case 0xB6: /* Fallthrough */ + case 0xB7: + /* Embedded one byte length map */ + { + callbacks->map_start(context, + _cbor_load_uint8(source) - 0xA0); /* 0xA0 offset */ + return result; + } + case 0xB8: + /* One byte length map */ + { + if (claim_bytes(1, source_size, &result)) { + callbacks->map_start(context, _cbor_load_uint8(source + 1)); + } + return result; + } + case 0xB9: + /* Two bytes length map */ + { + if (claim_bytes(2, source_size, &result)) { + callbacks->map_start(context, _cbor_load_uint16(source + 1)); + } + return result; + } + case 0xBA: + /* Four bytes length map */ + { + if (claim_bytes(4, source_size, &result)) { + callbacks->map_start(context, _cbor_load_uint32(source + 1)); + } + return result; + } + case 0xBB: + /* Eight bytes length map */ + { + if (claim_bytes(8, source_size, &result)) { + callbacks->map_start(context, _cbor_load_uint64(source + 1)); + } + return result; + } + case 0xBC: /* Fallthrough */ + case 0xBD: /* Fallthrough */ + case 0xBE: + /* Reserved */ + { return (struct cbor_decoder_result){.status = CBOR_DECODER_ERROR}; } + case 0xBF: + /* Indefinite length map */ + { + callbacks->indef_map_start(context); + return result; + } + /* See https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml for tag + * assignment. All well-formed tags are processed regardless of validity + * since maintaining the known mapping would be impractical. + * + * Moreover, even tags in the reserved "standard" range are not assigned + * but may get assigned in the future (see e.g. + * https://github.com/PJK/libcbor/issues/307), so processing all tags + * improves forward compatibility. + */ + case 0xC0: /* Fallthrough */ + case 0xC1: /* Fallthrough */ + case 0xC2: /* Fallthrough */ + case 0xC3: /* Fallthrough */ + case 0xC4: /* Fallthrough */ + case 0xC5: /* Fallthrough */ + case 0xC6: /* Fallthrough */ + case 0xC7: /* Fallthrough */ + case 0xC8: /* Fallthrough */ + case 0xC9: /* Fallthrough */ + case 0xCA: /* Fallthrough */ + case 0xCB: /* Fallthrough */ + case 0xCC: /* Fallthrough */ + case 0xCD: /* Fallthrough */ + case 0xCE: /* Fallthrough */ + case 0xCF: /* Fallthrough */ + case 0xD0: /* Fallthrough */ + case 0xD1: /* Fallthrough */ + case 0xD2: /* Fallthrough */ + case 0xD3: /* Fallthrough */ + case 0xD4: /* Fallthrough */ + case 0xD5: /* Fallthrough */ + case 0xD6: /* Fallthrough */ + case 0xD7: /* Fallthrough */ + { + callbacks->tag(context, (uint64_t)(_cbor_load_uint8(source) - + 0xC0)); /* 0xC0 offset */ + return result; + } + case 0xD8: /* 1B tag */ + { + if (claim_bytes(1, source_size, &result)) { + callbacks->tag(context, _cbor_load_uint8(source + 1)); + } + return result; + } + case 0xD9: /* 2B tag */ + { + if (claim_bytes(2, source_size, &result)) { + callbacks->tag(context, _cbor_load_uint16(source + 1)); + } + return result; + } + case 0xDA: /* 4B tag */ + { + if (claim_bytes(4, source_size, &result)) { + callbacks->tag(context, _cbor_load_uint32(source + 1)); + } + return result; + } + case 0xDB: /* 8B tag */ + { + if (claim_bytes(8, source_size, &result)) { + callbacks->tag(context, _cbor_load_uint64(source + 1)); + } + return result; + } + case 0xDC: /* Fallthrough */ + case 0xDD: /* Fallthrough */ + case 0xDE: /* Fallthrough */ + case 0xDF: /* Reserved */ + { + return (struct cbor_decoder_result){.status = CBOR_DECODER_ERROR}; + } + case 0xE0: /* Fallthrough */ + case 0xE1: /* Fallthrough */ + case 0xE2: /* Fallthrough */ + case 0xE3: /* Fallthrough */ + case 0xE4: /* Fallthrough */ + case 0xE5: /* Fallthrough */ + case 0xE6: /* Fallthrough */ + case 0xE7: /* Fallthrough */ + case 0xE8: /* Fallthrough */ + case 0xE9: /* Fallthrough */ + case 0xEA: /* Fallthrough */ + case 0xEB: /* Fallthrough */ + case 0xEC: /* Fallthrough */ + case 0xED: /* Fallthrough */ + case 0xEE: /* Fallthrough */ + case 0xEF: /* Fallthrough */ + case 0xF0: /* Fallthrough */ + case 0xF1: /* Fallthrough */ + case 0xF2: /* Fallthrough */ + case 0xF3: /* Simple value - unassigned */ + { + return (struct cbor_decoder_result){.status = CBOR_DECODER_ERROR}; + } + case 0xF4: + /* False */ + { + callbacks->boolean(context, false); + return result; + } + case 0xF5: + /* True */ + { + callbacks->boolean(context, true); + return result; + } + case 0xF6: + /* Null */ + { + callbacks->null(context); + return result; + } + case 0xF7: + /* Undefined */ + { + callbacks->undefined(context); + return result; + } + case 0xF8: + /* 1B simple value, unassigned */ + { return (struct cbor_decoder_result){.status = CBOR_DECODER_ERROR}; } + case 0xF9: + /* 2B float */ + { + if (claim_bytes(2, source_size, &result)) { + callbacks->float2(context, _cbor_load_half(source + 1)); + } + return result; + } + case 0xFA: + /* 4B float */ + { + if (claim_bytes(4, source_size, &result)) { + callbacks->float4(context, _cbor_load_float(source + 1)); + } + return result; + } + case 0xFB: + /* 8B float */ + { + if (claim_bytes(8, source_size, &result)) { + callbacks->float8(context, _cbor_load_double(source + 1)); + } + return result; + } + case 0xFC: /* Fallthrough */ + case 0xFD: /* Fallthrough */ + case 0xFE: + /* Reserved */ + { return (struct cbor_decoder_result){.status = CBOR_DECODER_ERROR}; } + case 0xFF: + /* Break */ + callbacks->indef_break(context); + // Never happens, the switch statement is exhaustive on the 1B range; make + // compiler happy + default: + return result; + } +} diff --git a/source/external/libcbor/cbor/streaming.h b/source/external/libcbor/cbor/streaming.h new file mode 100644 index 000000000..cb908e17e --- /dev/null +++ b/source/external/libcbor/cbor/streaming.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_STREAMING_H +#define LIBCBOR_STREAMING_H + +#include "callbacks.h" +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Stateless decoder + * + * Will try parsing the \p source and will invoke the appropriate callback on + * success. Decodes one item at a time. No memory allocations occur. + * + * @param source Input buffer + * @param source_size Length of the buffer + * @param callbacks The callback bundle + * @param context An arbitrary pointer to allow for maintaining context. + */ +_CBOR_NODISCARD CBOR_EXPORT struct cbor_decoder_result cbor_stream_decode( + cbor_data source, size_t source_size, + const struct cbor_callbacks* callbacks, void* context); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_STREAMING_H diff --git a/source/external/libcbor/cbor/strings.c b/source/external/libcbor/cbor/strings.c new file mode 100644 index 000000000..6ae96545c --- /dev/null +++ b/source/external/libcbor/cbor/strings.c @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "strings.h" +#include +#include "internal/memory_utils.h" +#include "internal/unicode.h" + +cbor_item_t *cbor_new_definite_string(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t)); + _CBOR_NOTNULL(item); + *item = (cbor_item_t){ + .refcount = 1, + .type = CBOR_TYPE_STRING, + .metadata = {.string_metadata = {_CBOR_METADATA_DEFINITE, 0}}}; + return item; +} + +cbor_item_t *cbor_new_indefinite_string(void) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t)); + _CBOR_NOTNULL(item); + *item = (cbor_item_t){ + .refcount = 1, + .type = CBOR_TYPE_STRING, + .metadata = {.string_metadata = {.type = _CBOR_METADATA_INDEFINITE, + .length = 0}}, + .data = _cbor_malloc(sizeof(struct cbor_indefinite_string_data))}; + _CBOR_DEPENDENT_NOTNULL(item, item->data); + *((struct cbor_indefinite_string_data *)item->data) = + (struct cbor_indefinite_string_data){ + .chunk_count = 0, + .chunk_capacity = 0, + .chunks = NULL, + }; + return item; +} + +cbor_item_t *cbor_build_string(const char *val) { + cbor_item_t *item = cbor_new_definite_string(); + _CBOR_NOTNULL(item); + size_t len = strlen(val); + unsigned char *handle = _cbor_malloc(len); + _CBOR_DEPENDENT_NOTNULL(item, handle); + memcpy(handle, val, len); + cbor_string_set_handle(item, handle, len); + return item; +} + +cbor_item_t *cbor_build_stringn(const char *val, size_t length) { + cbor_item_t *item = cbor_new_definite_string(); + _CBOR_NOTNULL(item); + unsigned char *handle = _cbor_malloc(length); + _CBOR_DEPENDENT_NOTNULL(item, handle); + memcpy(handle, val, length); + cbor_string_set_handle(item, handle, length); + return item; +} + +void cbor_string_set_handle(cbor_item_t *item, + cbor_mutable_data CBOR_RESTRICT_POINTER data, + size_t length) { + CBOR_ASSERT(cbor_isa_string(item)); + CBOR_ASSERT(cbor_string_is_definite(item)); + item->data = data; + item->metadata.string_metadata.length = length; + struct _cbor_unicode_status unicode_status; + size_t codepoint_count = + _cbor_unicode_codepoint_count(data, length, &unicode_status); + CBOR_ASSERT(codepoint_count <= length); + if (unicode_status.status == _CBOR_UNICODE_OK) { + item->metadata.string_metadata.codepoint_count = codepoint_count; + } else { + item->metadata.string_metadata.codepoint_count = 0; + } +} + +cbor_item_t **cbor_string_chunks_handle(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_string(item)); + CBOR_ASSERT(cbor_string_is_indefinite(item)); + return ((struct cbor_indefinite_string_data *)item->data)->chunks; +} + +size_t cbor_string_chunk_count(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_string(item)); + CBOR_ASSERT(cbor_string_is_indefinite(item)); + return ((struct cbor_indefinite_string_data *)item->data)->chunk_count; +} + +bool cbor_string_add_chunk(cbor_item_t *item, cbor_item_t *chunk) { + CBOR_ASSERT(cbor_isa_string(item)); + CBOR_ASSERT(cbor_string_is_indefinite(item)); + struct cbor_indefinite_string_data *data = + (struct cbor_indefinite_string_data *)item->data; + if (data->chunk_count == data->chunk_capacity) { + if (!_cbor_safe_to_multiply(CBOR_BUFFER_GROWTH, data->chunk_capacity)) { + return false; + } + + size_t new_chunk_capacity = + data->chunk_capacity == 0 ? 1 + : CBOR_BUFFER_GROWTH * (data->chunk_capacity); + cbor_item_t **new_chunks_data = _cbor_realloc_multiple( + data->chunks, sizeof(cbor_item_t *), new_chunk_capacity); + + if (new_chunks_data == NULL) { + return false; + } + + data->chunk_capacity = new_chunk_capacity; + data->chunks = new_chunks_data; + } + data->chunks[data->chunk_count++] = cbor_incref(chunk); + return true; +} + +size_t cbor_string_length(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_string(item)); + return item->metadata.string_metadata.length; +} + +unsigned char *cbor_string_handle(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_string(item)); + return item->data; +} + +size_t cbor_string_codepoint_count(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_string(item)); + return item->metadata.string_metadata.codepoint_count; +} + +bool cbor_string_is_definite(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_string(item)); + return item->metadata.string_metadata.type == _CBOR_METADATA_DEFINITE; +} + +bool cbor_string_is_indefinite(const cbor_item_t *item) { + return !cbor_string_is_definite(item); +} diff --git a/source/external/libcbor/cbor/strings.h b/source/external/libcbor/cbor/strings.h new file mode 100644 index 000000000..3e03f8138 --- /dev/null +++ b/source/external/libcbor/cbor/strings.h @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_STRINGS_H +#define LIBCBOR_STRINGS_H + +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * ============================================================================ + * String manipulation + * ============================================================================ + */ + +/** Returns the length of the underlying string in bytes + * + * There can be fewer unicode character than bytes (see + * `cbor_string_codepoint_count`). For definite strings only. + * + * @param item a definite string + * @return length of the string. Zero if no chunk has been attached yet + */ +_CBOR_NODISCARD CBOR_EXPORT size_t cbor_string_length(const cbor_item_t *item); + +/** The number of codepoints in this string + * + * Might differ from `cbor_string_length` if there are multibyte codepoints. + * If the string data is not valid UTF-8, returns 0. + * + * @param item A string + * @return The number of codepoints in this string + */ +_CBOR_NODISCARD CBOR_EXPORT size_t +cbor_string_codepoint_count(const cbor_item_t *item); + +/** Is the string definite? + * + * @param item a string + * @return Is the string definite? + */ +_CBOR_NODISCARD CBOR_EXPORT bool cbor_string_is_definite( + const cbor_item_t *item); + +/** Is the string indefinite? + * + * @param item a string + * @return Is the string indefinite? + */ +_CBOR_NODISCARD CBOR_EXPORT bool cbor_string_is_indefinite( + const cbor_item_t *item); + +/** Get the handle to the underlying string + * + * Definite items only. Modifying the data is allowed. In that case, the caller + * takes responsibility for the effect on items this item might be a part of + * + * @param item A definite string + * @return The address of the underlying string. + * @return `NULL` if no data have been assigned yet. + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_mutable_data +cbor_string_handle(const cbor_item_t *item); + +/** Set the handle to the underlying string + * + * The data is assumed to be a valid UTF-8 string. If the string is non-empty + * and invalid, `cbor_string_codepoint_count` will return 0. + * + * \rst + * .. warning:: Using a pointer to a stack allocated constant is a common + * mistake. Lifetime of the string will expire when it goes out of scope and + * the CBOR item will be left inconsistent. + * \endrst + * + * @param item A definite string + * @param data The memory block. The caller gives up the ownership of the block. + * libcbor will deallocate it when appropriate using its free function + * @param length Length of the data block + */ +CBOR_EXPORT void cbor_string_set_handle( + cbor_item_t *item, cbor_mutable_data CBOR_RESTRICT_POINTER data, + size_t length); + +/** Get the handle to the array of chunks + * + * Manipulations with the memory block (e.g. sorting it) are allowed, but the + * validity and the number of chunks must be retained. + * + * @param item A indefinite string + * @return array of #cbor_string_chunk_count definite strings + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t **cbor_string_chunks_handle( + const cbor_item_t *item); + +/** Get the number of chunks this string consist of + * + * @param item A indefinite string + * @return The chunk count. 0 for freshly created items. + */ +_CBOR_NODISCARD CBOR_EXPORT size_t +cbor_string_chunk_count(const cbor_item_t *item); + +/** Appends a chunk to the string + * + * Indefinite strings only. + * + * May realloc the chunk storage. + * + * @param item An indefinite string + * @param chunk A definite string item. Its reference count will be increased + * by one. + * @return `true` on success. `false` on memory allocation failure. In that + * case, the refcount of @p `chunk` is not increased and the @p `item` is left + * intact. + */ +_CBOR_NODISCARD CBOR_EXPORT bool cbor_string_add_chunk(cbor_item_t *item, + cbor_item_t *chunk); + +/** Creates a new definite string + * + * The handle is initialized to `NULL` and length to 0 + * + * @return Reference to the new string item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_definite_string(void); + +/** Creates a new indefinite string + * + * The chunks array is initialized to `NULL` and chunkcount to 0 + * + * @return Reference to the new string item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_indefinite_string(void); + +/** Creates a new string and initializes it + * + * The data from `val` will be copied to a newly allocated memory block. + * + * Note that valid UTF-8 strings do not contain null bytes, so this routine is + * correct for all valid inputs. If the input is not guaranteed to be valid, + * use `cbor_build_stringn` instead. + * + * @param val A null-terminated UTF-8 string + * @return Reference to the new string item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_string(const char *val); + +/** Creates a new string and initializes it + * + * The data from `handle` will be copied to a newly allocated memory block. + * + * All @p `length` bytes will be stored in the string, even if there are null + * bytes or invalid UTF-8 sequences. + * + * @param val A UTF-8 string, at least @p `length` bytes long + * @param length Length (in bytes) of the string passed in @p `val`. + * @return Reference to the new string item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_stringn(const char *val, + size_t length); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_STRINGS_H diff --git a/source/external/libcbor/cbor/tags.c b/source/external/libcbor/cbor/tags.c new file mode 100644 index 000000000..3f3edb0b0 --- /dev/null +++ b/source/external/libcbor/cbor/tags.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include "tags.h" + +cbor_item_t *cbor_new_tag(uint64_t value) { + cbor_item_t *item = _cbor_malloc(sizeof(cbor_item_t)); + _CBOR_NOTNULL(item); + + *item = (cbor_item_t){ + .refcount = 1, + .type = CBOR_TYPE_TAG, + .metadata = {.tag_metadata = {.value = value, .tagged_item = NULL}}, + .data = NULL /* Never used */ + }; + return item; +} + +cbor_item_t *cbor_tag_item(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_tag(item)); + return cbor_incref(item->metadata.tag_metadata.tagged_item); +} + +uint64_t cbor_tag_value(const cbor_item_t *item) { + CBOR_ASSERT(cbor_isa_tag(item)); + return item->metadata.tag_metadata.value; +} + +void cbor_tag_set_item(cbor_item_t *item, cbor_item_t *tagged_item) { + CBOR_ASSERT(cbor_isa_tag(item)); + cbor_incref(tagged_item); + item->metadata.tag_metadata.tagged_item = tagged_item; +} + +cbor_item_t *cbor_build_tag(uint64_t value, cbor_item_t *item) { + cbor_item_t *res = cbor_new_tag(value); + if (res == NULL) { + return NULL; + } + cbor_tag_set_item(res, item); + return res; +} diff --git a/source/external/libcbor/cbor/tags.h b/source/external/libcbor/cbor/tags.h new file mode 100644 index 000000000..a7365df10 --- /dev/null +++ b/source/external/libcbor/cbor/tags.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2014-2020 Pavel Kalvoda + * + * libcbor is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIBCBOR_TAGS_H +#define LIBCBOR_TAGS_H + +#include "cbor/cbor_export.h" +#include "cbor/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * ============================================================================ + * Tag manipulation + * ============================================================================ + */ + +/** Create a new tag + * + * @param value The tag value. Please consult the tag repository + * @return Reference to the new tag item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_new_tag(uint64_t value); + +/** Get the tagged item + * + * @param item A tag + * @return Reference to the tagged item + * + * Increases the reference count of the underlying item. The returned reference + * must be released using #cbor_decref. + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_tag_item(const cbor_item_t *item); + +/** Get tag value + * + * @param item A tag + * @return The tag value. Please consult the tag repository + */ +_CBOR_NODISCARD CBOR_EXPORT uint64_t cbor_tag_value(const cbor_item_t *item); + +/** Set the tagged item + * + * @param item A tag + * @param tagged_item The item to tag. Its reference count will be increased + * by one. + */ +CBOR_EXPORT void cbor_tag_set_item(cbor_item_t *item, cbor_item_t *tagged_item); + +/** Build a new tag + * + * @param item The item to tag. Its reference count will be increased by + * one. + * @param value Tag value + * @return Reference to the new tag item. The item's reference count is + * initialized to one. + * @return `NULL` if memory allocation fails + */ +_CBOR_NODISCARD CBOR_EXPORT cbor_item_t *cbor_build_tag(uint64_t value, + cbor_item_t *item); + +#ifdef __cplusplus +} +#endif + +#endif // LIBCBOR_TAGS_H diff --git a/source/json.c b/source/json.c index 326354d62..28524a88c 100644 --- a/source/json.c +++ b/source/json.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include "external/cJSON.h" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ee281ac28..d3556c5f7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -282,6 +282,8 @@ add_test_case(test_byte_buf_append_lookup_failure) add_test_case(test_byte_buf_reserve) add_test_case(test_byte_buf_reserve_initial_capacity_zero) add_test_case(test_byte_buf_reserve_relative) +add_test_case(test_byte_buf_reserve_smart) +add_test_case(test_byte_buf_reserve_smart_relative) add_test_case(test_byte_buf_reset) add_test_case(test_byte_cursor_compare_lexical) add_test_case(test_byte_cursor_compare_lookup) @@ -525,6 +527,15 @@ add_test_case(test_cross_process_lock_invalid_nonce_fails) add_test_case(host_util_is_ipv4) add_test_case(host_util_is_ipv6) +add_test_case(cbor_encode_decode_int_test) +add_test_case(cbor_encode_decode_double_test) +add_test_case(cbor_encode_decode_bool_test) +add_test_case(cbor_encode_decode_bytesstr_str_test) +add_test_case(cbor_encode_decode_array_map_test) +add_test_case(cbor_encode_decode_simple_value_test) +add_test_case(cbor_encode_decode_indef_test) +add_test_case(cbor_decode_error_handling_test) + generate_test_driver(${PROJECT_NAME}-tests) if(NOT MSVC AND NOT LEGACY_COMPILER_SUPPORT) diff --git a/tests/byte_buf_test.c b/tests/byte_buf_test.c index 545ec4a0e..035653853 100644 --- a/tests/byte_buf_test.c +++ b/tests/byte_buf_test.c @@ -4,7 +4,7 @@ */ #include - +#include #include #include @@ -798,6 +798,63 @@ static int s_test_byte_buf_reserve_relative(struct aws_allocator *allocator, voi } AWS_TEST_CASE(test_byte_buf_reserve_relative, s_test_byte_buf_reserve_relative) +static int s_test_byte_buf_reserve_smart(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + (void)allocator; + + struct aws_byte_buf buffer; + size_t base = 10; + aws_byte_buf_init(&buffer, allocator, base); + ASSERT_TRUE(aws_byte_buf_reserve_smart(&buffer, 2 * base) == AWS_OP_SUCCESS); + ASSERT_UINT_EQUALS(2 * base, buffer.capacity); + ASSERT_TRUE(aws_byte_buf_reserve_smart(&buffer, 3 * base) == AWS_OP_SUCCESS); + /* double the previous capacity instead of just expand the capacity to meet the requirement to reduce the number of + * allocations */ + ASSERT_UINT_EQUALS(4 * base, buffer.capacity); + ASSERT_TRUE(aws_byte_buf_reserve_smart(&buffer, 5 * base) == AWS_OP_SUCCESS); + /* double the previous capacity instead of just expand the capacity to meet the requirement to reduce the number of + * allocations */ + ASSERT_UINT_EQUALS(8 * base, buffer.capacity); + ASSERT_TRUE(aws_byte_buf_reserve_smart(&buffer, 5 * base) == AWS_OP_SUCCESS); + ASSERT_UINT_EQUALS(8 * base, buffer.capacity); + + aws_byte_buf_clean_up(&buffer); + + return 0; +} +AWS_TEST_CASE(test_byte_buf_reserve_smart, s_test_byte_buf_reserve_smart) + +static int s_test_byte_buf_reserve_smart_relative(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + (void)allocator; + + struct aws_byte_buf buffer; + aws_byte_buf_init(&buffer, allocator, 1); + + struct aws_byte_cursor prefix_cursor = aws_byte_cursor_from_string(s_reserve_test_prefix); + size_t length = prefix_cursor.len; + + ASSERT_TRUE(aws_byte_buf_reserve_smart_relative(&buffer, length) == AWS_OP_SUCCESS); + ASSERT_TRUE(aws_byte_buf_append(&buffer, &prefix_cursor) == AWS_OP_SUCCESS); + ASSERT_UINT_EQUALS(length, buffer.capacity); + ASSERT_TRUE(aws_byte_buf_reserve_smart_relative(&buffer, length) == AWS_OP_SUCCESS); + ASSERT_TRUE(aws_byte_buf_append(&buffer, &prefix_cursor) == AWS_OP_SUCCESS); + ASSERT_UINT_EQUALS(2 * length, buffer.capacity); + ASSERT_TRUE(aws_byte_buf_reserve_smart_relative(&buffer, length) == AWS_OP_SUCCESS); + ASSERT_TRUE(aws_byte_buf_append(&buffer, &prefix_cursor) == AWS_OP_SUCCESS); + /* 4 times base as it's expanded to twice the original capacity to prevent too many allocation */ + ASSERT_UINT_EQUALS(4 * length, buffer.capacity); + ASSERT_TRUE(aws_byte_buf_reserve_smart_relative(&buffer, length) == AWS_OP_SUCCESS); + ASSERT_TRUE(aws_byte_buf_append(&buffer, &prefix_cursor) == AWS_OP_SUCCESS); + /* still 4 times base as it doesn't need to expand again */ + ASSERT_UINT_EQUALS(4 * length, buffer.capacity); + + aws_byte_buf_clean_up(&buffer); + + return 0; +} +AWS_TEST_CASE(test_byte_buf_reserve_smart_relative, s_test_byte_buf_reserve_smart_relative) + static int s_test_byte_cursor_starts_with(struct aws_allocator *allocator, void *ctx) { (void)ctx; (void)allocator; diff --git a/tests/cbor_test.c b/tests/cbor_test.c new file mode 100644 index 000000000..585ffcca1 --- /dev/null +++ b/tests/cbor_test.c @@ -0,0 +1,489 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include +#include + +#include + +#define CBOR_TEST_CASE(NAME) \ + AWS_TEST_CASE(NAME, s_test_##NAME); \ + static int s_test_##NAME(struct aws_allocator *allocator, void *ctx) + +CBOR_TEST_CASE(cbor_encode_decode_int_test) { + (void)allocator; + (void)ctx; + aws_common_library_init(allocator); + enum { VALUE_NUM = 6 }; + + /** + * Less than 24 only take 1 byte, + * 24 to uint8_t max takes 2 bytes + * uint8_t max to uint16_t max takes 3 bytes + * uint16_t max to uint32_t maxx takes 5 bytes + * uint32_t max to uint64_t max takes 9 bytes + */ + uint64_t values[VALUE_NUM] = {23, 24, UINT8_MAX + 1, UINT16_MAX + 1U, UINT32_MAX + 1LLU, UINT64_MAX}; + uint64_t expected_encoded_len[VALUE_NUM] = {1, 2, 3, 5, 9, 9}; + + size_t encoded_len = 0; + struct aws_cbor_encoder *encoder = aws_cbor_encoder_new(allocator); + /* Unsigned int */ + for (size_t i = 0; i < VALUE_NUM; i++) { + aws_cbor_encoder_write_uint(encoder, values[i]); + struct aws_byte_cursor cursor = aws_cbor_encoder_get_encoded_data(encoder); + ASSERT_UINT_EQUALS(encoded_len + expected_encoded_len[i], cursor.len); + encoded_len = cursor.len; + } + /* Negative int */ + for (size_t i = 0; i < VALUE_NUM; i++) { + aws_cbor_encoder_write_negint(encoder, values[i]); + struct aws_byte_cursor cursor = aws_cbor_encoder_get_encoded_data(encoder); + ASSERT_UINT_EQUALS(encoded_len + expected_encoded_len[i], cursor.len); + encoded_len = cursor.len; + } + struct aws_byte_cursor final_cursor = aws_cbor_encoder_get_encoded_data(encoder); + struct aws_cbor_decoder *decoder = aws_cbor_decoder_new(allocator, final_cursor); + /* Unsigned int */ + for (size_t i = 0; i < VALUE_NUM; i++) { + uint64_t result; + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_unsigned_int_val(decoder, &result)); + ASSERT_UINT_EQUALS(values[i], result); + } + /* Negative int */ + for (size_t i = 0; i < VALUE_NUM; i++) { + uint64_t result; + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_negative_int_val(decoder, &result)); + ASSERT_UINT_EQUALS(values[i], result); + } + + ASSERT_UINT_EQUALS(0, aws_cbor_decoder_get_remaining_length(decoder)); + + aws_cbor_encoder_destroy(encoder); + aws_cbor_decoder_destroy(decoder); + aws_common_library_clean_up(); + return AWS_OP_SUCCESS; +} + +CBOR_TEST_CASE(cbor_encode_decode_double_test) { + (void)allocator; + (void)ctx; + aws_common_library_init(allocator); + enum { VALUE_NUM = 10 }; + + /** + * 1 as unsigned int, takes 1 byte + * -1 as negative int, takes 1 byte + * 1.1 double, takes 9 bytes + * 1.1f is float, takes 5 bytes + * -1.1f is float, takes 5 bytes + * INFINITY will be float, takes 5 bytes + * FLT_MAX still a float, take 5 bytes + * DBL_MAX will be a double takes 9 bytes + * DBL_MIN will be a double takes 9 bytes + * HUGE_VAL + */ + double values[VALUE_NUM] = {1.0, -1.0, 1.1, 1.1f, -1.1f, INFINITY, FLT_MAX, DBL_MAX, DBL_MIN, HUGE_VAL}; + uint64_t expected_encoded_len[VALUE_NUM] = {1, 1, 9, 5, 5, 5, 5, 9, 9, 5}; + + int expected_encoded_type[VALUE_NUM] = { + AWS_CBOR_TYPE_UINT, + AWS_CBOR_TYPE_NEGINT, + AWS_CBOR_TYPE_FLOAT, + AWS_CBOR_TYPE_FLOAT, + AWS_CBOR_TYPE_FLOAT, + AWS_CBOR_TYPE_FLOAT, + AWS_CBOR_TYPE_FLOAT, + AWS_CBOR_TYPE_FLOAT, + AWS_CBOR_TYPE_FLOAT, + AWS_CBOR_TYPE_FLOAT, + }; + + size_t encoded_len = 0; + struct aws_cbor_encoder *encoder = aws_cbor_encoder_new(allocator); + for (size_t i = 0; i < VALUE_NUM; i++) { + aws_cbor_encoder_write_float(encoder, values[i]); + struct aws_byte_cursor cursor = aws_cbor_encoder_get_encoded_data(encoder); + ASSERT_UINT_EQUALS(encoded_len + expected_encoded_len[i], cursor.len); + encoded_len = cursor.len; + } + struct aws_byte_cursor final_cursor = aws_cbor_encoder_get_encoded_data(encoder); + struct aws_cbor_decoder *decoder = aws_cbor_decoder_new(allocator, final_cursor); + /* Unsigned int, 1 */ + size_t index = 0; + uint64_t result = 0; + enum aws_cbor_type out_type = AWS_CBOR_TYPE_UNKNOWN; + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, expected_encoded_type[index]); + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_unsigned_int_val(decoder, &result)); + ASSERT_TRUE(values[index++] == result); + /* negative int, -1 */ + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, expected_encoded_type[index]); + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_negative_int_val(decoder, &result)); + /* Convert the decode val to expected val. */ + ASSERT_TRUE((-1 - values[index++]) == result); + /* 1.1 double */ + double double_result = 0; + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, expected_encoded_type[index]); + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_float_val(decoder, &double_result)); + ASSERT_TRUE(values[index++] == double_result); + /* 1.1 float */ + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, expected_encoded_type[index]); + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_float_val(decoder, &double_result)); + ASSERT_TRUE(values[index++] == double_result); + /* -1.1 float */ + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, expected_encoded_type[index]); + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_float_val(decoder, &double_result)); + ASSERT_TRUE(values[index++] == double_result); + /* INFINITY */ + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, expected_encoded_type[index]); + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_float_val(decoder, &double_result)); + ASSERT_TRUE(values[index++] == double_result); + /* FLT_MAX */ + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, expected_encoded_type[index]); + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_float_val(decoder, &double_result)); + ASSERT_TRUE(values[index++] == double_result); + /* DBL_MAX */ + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, expected_encoded_type[index]); + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_float_val(decoder, &double_result)); + ASSERT_TRUE(values[index++] == double_result); + /* DBL_MIN */ + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, expected_encoded_type[index]); + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_float_val(decoder, &double_result)); + ASSERT_TRUE(values[index++] == double_result); + /* HUGE_VAL */ + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, expected_encoded_type[index]); + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_float_val(decoder, &double_result)); + ASSERT_TRUE(values[index++] == double_result); + + ASSERT_UINT_EQUALS(0, aws_cbor_decoder_get_remaining_length(decoder)); + + aws_cbor_encoder_destroy(encoder); + aws_cbor_decoder_destroy(decoder); + aws_common_library_clean_up(); + return AWS_OP_SUCCESS; +} + +CBOR_TEST_CASE(cbor_encode_decode_bool_test) { + (void)allocator; + (void)ctx; + aws_common_library_init(allocator); + enum { VALUE_NUM = 2 }; + bool values[VALUE_NUM] = {true, false}; + uint64_t expected_encoded_len[VALUE_NUM] = {1, 1}; + + size_t encoded_len = 0; + struct aws_cbor_encoder *encoder = aws_cbor_encoder_new(allocator); + for (size_t i = 0; i < VALUE_NUM; i++) { + aws_cbor_encoder_write_bool(encoder, values[i]); + struct aws_byte_cursor cursor = aws_cbor_encoder_get_encoded_data(encoder); + ASSERT_UINT_EQUALS(encoded_len + expected_encoded_len[i], cursor.len); + encoded_len = cursor.len; + } + struct aws_byte_cursor final_cursor = aws_cbor_encoder_get_encoded_data(encoder); + struct aws_cbor_decoder *decoder = aws_cbor_decoder_new(allocator, final_cursor); + for (size_t i = 0; i < VALUE_NUM; i++) { + bool result; + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_boolean_val(decoder, &result)); + ASSERT_UINT_EQUALS(values[i], result); + } + + ASSERT_UINT_EQUALS(0, aws_cbor_decoder_get_remaining_length(decoder)); + + aws_cbor_encoder_destroy(encoder); + aws_cbor_decoder_destroy(decoder); + aws_common_library_clean_up(); + return AWS_OP_SUCCESS; +} + +CBOR_TEST_CASE(cbor_encode_decode_bytesstr_str_test) { + (void)allocator; + (void)ctx; + aws_common_library_init(allocator); + struct aws_byte_cursor val_1 = aws_byte_cursor_from_c_str("my test"); + struct aws_byte_cursor val_2 = aws_byte_cursor_from_c_str("write more tests"); + + enum { VALUE_NUM = 2 }; + struct aws_byte_cursor values[VALUE_NUM] = {val_1, val_2}; + uint64_t expected_encoded_len[VALUE_NUM] = {1 + val_1.len, 1 + val_2.len}; + + size_t encoded_len = 0; + struct aws_cbor_encoder *encoder = aws_cbor_encoder_new(allocator); + for (size_t i = 0; i < VALUE_NUM; i++) { + aws_cbor_encoder_write_text(encoder, values[i]); + struct aws_byte_cursor cursor = aws_cbor_encoder_get_encoded_data(encoder); + ASSERT_UINT_EQUALS(encoded_len + expected_encoded_len[i], cursor.len); + encoded_len = cursor.len; + } + for (size_t i = 0; i < VALUE_NUM; i++) { + aws_cbor_encoder_write_bytes(encoder, values[i]); + struct aws_byte_cursor cursor = aws_cbor_encoder_get_encoded_data(encoder); + ASSERT_UINT_EQUALS(encoded_len + expected_encoded_len[i], cursor.len); + encoded_len = cursor.len; + } + struct aws_byte_cursor final_cursor = aws_cbor_encoder_get_encoded_data(encoder); + struct aws_cbor_decoder *decoder = aws_cbor_decoder_new(allocator, final_cursor); + for (size_t i = 0; i < VALUE_NUM; i++) { + struct aws_byte_cursor result; + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_text_val(decoder, &result)); + ASSERT_TRUE(aws_byte_cursor_eq(&result, &values[i])); + } + for (size_t i = 0; i < VALUE_NUM; i++) { + struct aws_byte_cursor result; + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_bytes_val(decoder, &result)); + ASSERT_TRUE(aws_byte_cursor_eq(&result, &values[i])); + } + + ASSERT_UINT_EQUALS(0, aws_cbor_decoder_get_remaining_length(decoder)); + + aws_cbor_encoder_destroy(encoder); + aws_cbor_decoder_destroy(decoder); + aws_common_library_clean_up(); + return AWS_OP_SUCCESS; +} + +CBOR_TEST_CASE(cbor_encode_decode_array_map_test) { + (void)allocator; + (void)ctx; + aws_common_library_init(allocator); + struct aws_byte_cursor val_1 = aws_byte_cursor_from_c_str("my test"); + struct aws_byte_cursor val_2 = aws_byte_cursor_from_c_str("write more tests"); + + enum { VALUE_NUM = 2 }; + struct aws_byte_cursor values[VALUE_NUM] = {val_1, val_2}; + uint64_t expected_encoded_len[VALUE_NUM] = {1 + val_1.len, 1 + val_2.len}; + + size_t encoded_len = 0; + struct aws_cbor_encoder *encoder = aws_cbor_encoder_new(allocator); + + /* Array with 2 elements */ + aws_cbor_encoder_write_array_start(encoder, 2); + struct aws_byte_cursor encoded_cursor = aws_cbor_encoder_get_encoded_data(encoder); + /* Array start with 2 element only takes 1 byte */ + ASSERT_UINT_EQUALS(encoded_len + 1, encoded_cursor.len); + encoded_len = encoded_cursor.len; + + for (size_t i = 0; i < VALUE_NUM; i++) { + aws_cbor_encoder_write_text(encoder, values[i]); + struct aws_byte_cursor cursor = aws_cbor_encoder_get_encoded_data(encoder); + ASSERT_UINT_EQUALS(encoded_len + expected_encoded_len[i], cursor.len); + encoded_len = cursor.len; + } + + /* Map with 1 element */ + aws_cbor_encoder_write_map_start(encoder, 1); + encoded_cursor = aws_cbor_encoder_get_encoded_data(encoder); + /* Map start with 1 (key, value pair) only takes 1 byte */ + ASSERT_UINT_EQUALS(encoded_len + 1, encoded_cursor.len); + encoded_len = encoded_cursor.len; + for (size_t i = 0; i < VALUE_NUM; i++) { + aws_cbor_encoder_write_bytes(encoder, values[i]); + struct aws_byte_cursor cursor = aws_cbor_encoder_get_encoded_data(encoder); + ASSERT_UINT_EQUALS(encoded_len + expected_encoded_len[i], cursor.len); + encoded_len = cursor.len; + } + + /* Map with a lot element, not closure. */ + aws_cbor_encoder_write_array_start(encoder, UINT16_MAX + 1); + encoded_cursor = aws_cbor_encoder_get_encoded_data(encoder); + /* The size takes 4 bytes and one more for the cbor start item */ + ASSERT_UINT_EQUALS(encoded_len + 5, encoded_cursor.len); + encoded_len = encoded_cursor.len; + + aws_cbor_encoder_write_map_start(encoder, UINT16_MAX + 1); + encoded_cursor = aws_cbor_encoder_get_encoded_data(encoder); + /* The size takes 4 bytes and one more for the cbor start item */ + ASSERT_UINT_EQUALS(encoded_len + 5, encoded_cursor.len); + encoded_len = encoded_cursor.len; + + struct aws_byte_cursor final_cursor = aws_cbor_encoder_get_encoded_data(encoder); + struct aws_cbor_decoder *decoder = aws_cbor_decoder_new(allocator, final_cursor); + + uint64_t element_size = 0; + aws_cbor_decoder_pop_next_array_start(decoder, &element_size); + ASSERT_UINT_EQUALS(element_size, 2); + for (size_t i = 0; i < VALUE_NUM; i++) { + struct aws_byte_cursor result; + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_text_val(decoder, &result)); + ASSERT_TRUE(aws_byte_cursor_eq(&result, &values[i])); + } + aws_cbor_decoder_pop_next_map_start(decoder, &element_size); + ASSERT_UINT_EQUALS(element_size, 1); + for (size_t i = 0; i < VALUE_NUM; i++) { + struct aws_byte_cursor result; + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_bytes_val(decoder, &result)); + ASSERT_TRUE(aws_byte_cursor_eq(&result, &values[i])); + } + aws_cbor_decoder_pop_next_array_start(decoder, &element_size); + ASSERT_UINT_EQUALS(element_size, UINT16_MAX + 1); + aws_cbor_decoder_pop_next_map_start(decoder, &element_size); + ASSERT_UINT_EQUALS(element_size, UINT16_MAX + 1); + + ASSERT_UINT_EQUALS(0, aws_cbor_decoder_get_remaining_length(decoder)); + + aws_cbor_encoder_destroy(encoder); + aws_cbor_decoder_destroy(decoder); + aws_common_library_clean_up(); + return AWS_OP_SUCCESS; +} + +CBOR_TEST_CASE(cbor_encode_decode_simple_value_test) { + (void)allocator; + (void)ctx; + aws_common_library_init(allocator); + + struct aws_cbor_encoder *encoder = aws_cbor_encoder_new(allocator); + aws_cbor_encoder_write_null(encoder); + aws_cbor_encoder_write_undefined(encoder); + struct aws_byte_cursor final_cursor = aws_cbor_encoder_get_encoded_data(encoder); + /* in total 2 bytes for two simple value */ + ASSERT_UINT_EQUALS(2, final_cursor.len); + struct aws_cbor_decoder *decoder = aws_cbor_decoder_new(allocator, final_cursor); + + enum aws_cbor_type out_type = AWS_CBOR_TYPE_UNKNOWN; + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, AWS_CBOR_TYPE_NULL); + ASSERT_SUCCESS(aws_cbor_decoder_consume_next_single_element(decoder)); + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, AWS_CBOR_TYPE_UNDEFINED); + ASSERT_SUCCESS(aws_cbor_decoder_consume_next_single_element(decoder)); + ASSERT_UINT_EQUALS(0, aws_cbor_decoder_get_remaining_length(decoder)); + + aws_cbor_encoder_destroy(encoder); + aws_cbor_decoder_destroy(decoder); + aws_common_library_clean_up(); + return AWS_OP_SUCCESS; +} + +/* Test a complicate multiple stacks encode and decode */ +CBOR_TEST_CASE(cbor_encode_decode_indef_test) { + (void)allocator; + (void)ctx; + aws_common_library_init(allocator); + struct aws_byte_cursor val_1 = aws_byte_cursor_from_c_str("my test"); + struct aws_byte_cursor val_2 = aws_byte_cursor_from_c_str("write more tests"); + + struct aws_cbor_encoder *encoder = aws_cbor_encoder_new(allocator); + + /* Create a non-sense stack of inf collections. */ + aws_cbor_encoder_write_indef_map_start(encoder); + /* Key */ + aws_cbor_encoder_write_text(encoder, val_1); + /* Value */ + aws_cbor_encoder_write_indef_array_start(encoder); + /* element 1 in array */ + aws_cbor_encoder_write_indef_text_start(encoder); + aws_cbor_encoder_write_text(encoder, val_1); + aws_cbor_encoder_write_text(encoder, val_2); + aws_cbor_encoder_write_break(encoder); + /* element 2 in array */ + aws_cbor_encoder_write_indef_bytes_start(encoder); + aws_cbor_encoder_write_bytes(encoder, val_1); + aws_cbor_encoder_write_bytes(encoder, val_2); + aws_cbor_encoder_write_break(encoder); + /* element 3 as a tag in array */ + aws_cbor_encoder_write_tag(encoder, AWS_CBOR_TAG_DECIMAL_FRACTION); + aws_cbor_encoder_write_indef_array_start(encoder); + aws_cbor_encoder_write_indef_bytes_start(encoder); + aws_cbor_encoder_write_bytes(encoder, val_1); + aws_cbor_encoder_write_bytes(encoder, val_2); + aws_cbor_encoder_write_break(encoder); + aws_cbor_encoder_write_break(encoder); + /* Closure for the array */ + aws_cbor_encoder_write_break(encoder); + /* Closure for the map */ + aws_cbor_encoder_write_break(encoder); + + struct aws_byte_cursor final_cursor = aws_cbor_encoder_get_encoded_data(encoder); + struct aws_cbor_decoder *decoder = aws_cbor_decoder_new(allocator, final_cursor); + + enum aws_cbor_type out_type = AWS_CBOR_TYPE_UNKNOWN; + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(out_type, AWS_CBOR_TYPE_INDEF_MAP_START); + + /* Get rid of the whole inf map with all the data content */ + ASSERT_SUCCESS(aws_cbor_decoder_consume_next_whole_data_item(decoder)); + + ASSERT_UINT_EQUALS(0, aws_cbor_decoder_get_remaining_length(decoder)); + + aws_cbor_encoder_destroy(encoder); + aws_cbor_decoder_destroy(decoder); + aws_common_library_clean_up(); + return AWS_OP_SUCCESS; +} + +CBOR_TEST_CASE(cbor_decode_error_handling_test) { + (void)allocator; + (void)ctx; + aws_common_library_init(allocator); + /* Major type 7 with argument 30, 11111110, malformed CBOR */ + uint8_t invalid_data[] = {0xFE}; + struct aws_byte_cursor invalid_cbor = aws_byte_cursor_from_array(invalid_data, sizeof(invalid_data)); + + enum aws_cbor_type out_type = AWS_CBOR_TYPE_UNKNOWN; + + /* 1. Malformed cbor data */ + struct aws_cbor_decoder *decoder = aws_cbor_decoder_new(allocator, invalid_cbor); + ASSERT_FAILS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(AWS_ERROR_INVALID_CBOR, aws_last_error()); + aws_cbor_decoder_destroy(decoder); + + /* 2. Empty cursor */ + struct aws_byte_cursor empty = {0}; + decoder = aws_cbor_decoder_new(allocator, empty); + ASSERT_FAILS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(AWS_ERROR_INVALID_CBOR, aws_last_error()); + aws_cbor_decoder_destroy(decoder); + + /* 3. Try get wrong type */ + struct aws_cbor_encoder *encoder = aws_cbor_encoder_new(allocator); + uint64_t val = 1; + aws_cbor_encoder_write_uint(encoder, val); + struct aws_byte_cursor final_cursor = aws_cbor_encoder_get_encoded_data(encoder); + decoder = aws_cbor_decoder_new(allocator, final_cursor); + uint64_t out = 0; + ASSERT_FAILS(aws_cbor_decoder_pop_next_array_start(decoder, &out)); + ASSERT_UINT_EQUALS(AWS_ERROR_CBOR_UNEXPECTED_TYPE, aws_last_error()); + /* But, we can still keep decoding for the right type */ + ASSERT_SUCCESS(aws_cbor_decoder_pop_next_unsigned_int_val(decoder, &out)); + ASSERT_UINT_EQUALS(val, out); + /* All the data has been consumed, now it's invalid */ + ASSERT_FAILS(aws_cbor_decoder_consume_next_whole_data_item(decoder)); + ASSERT_FAILS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(AWS_ERROR_INVALID_CBOR, aws_last_error()); + aws_cbor_decoder_destroy(decoder); + + /* 4. Consume data items with size */ + struct aws_byte_cursor val_1 = aws_byte_cursor_from_c_str("my test"); + aws_cbor_encoder_reset(encoder); + aws_cbor_encoder_write_map_start(encoder, 1); + /* Key */ + aws_cbor_encoder_write_text(encoder, val_1); + /* Value */ + aws_cbor_encoder_write_array_start(encoder, 1); + aws_cbor_encoder_write_tag(encoder, AWS_CBOR_TAG_NEGATIVE_BIGNUM); + aws_cbor_encoder_write_bytes(encoder, val_1); + final_cursor = aws_cbor_encoder_get_encoded_data(encoder); + decoder = aws_cbor_decoder_new(allocator, final_cursor); + ASSERT_SUCCESS(aws_cbor_decoder_peek_type(decoder, &out_type)); + ASSERT_UINT_EQUALS(AWS_CBOR_TYPE_MAP_START, out_type); + ASSERT_SUCCESS(aws_cbor_decoder_consume_next_whole_data_item(decoder)); + ASSERT_UINT_EQUALS(0, aws_cbor_decoder_get_remaining_length(decoder)); + aws_cbor_decoder_destroy(decoder); + + aws_cbor_encoder_destroy(encoder); + aws_common_library_clean_up(); + return AWS_OP_SUCCESS; +} diff --git a/tests/fuzz/cbor_decoding_transitive.c b/tests/fuzz/cbor_decoding_transitive.c new file mode 100644 index 000000000..3815db6cf --- /dev/null +++ b/tests/fuzz/cbor_decoding_transitive.c @@ -0,0 +1,87 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include + +/* NOLINTNEXTLINE(readability-identifier-naming) */ +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + struct aws_allocator *allocator = aws_mem_tracer_new(aws_default_allocator(), NULL, AWS_MEMTRACE_BYTES, 0); + struct aws_byte_cursor input = aws_byte_cursor_from_array(data, size); + aws_common_library_init(allocator); + + struct aws_cbor_decoder *decoder = aws_cbor_decoder_new(allocator, input); + union { + uint64_t unsigned_val; + uint64_t neg_val; + double double_val; + uint64_t tag_val; + bool boolean_val; + struct aws_byte_cursor str_val; + struct aws_byte_cursor bytes_val; + uint64_t map_start; + uint64_t array_start; + } cbor_data; + + enum aws_cbor_type out_type = AWS_CBOR_TYPE_UNKNOWN; + while (aws_cbor_decoder_peek_type(decoder, &out_type) == AWS_OP_SUCCESS) { + switch (out_type) { + case AWS_CBOR_TYPE_UINT: + AWS_FATAL_ASSERT( + aws_cbor_decoder_pop_next_unsigned_int_val(decoder, &cbor_data.unsigned_val) == AWS_OP_SUCCESS); + break; + case AWS_CBOR_TYPE_NEGINT: + AWS_FATAL_ASSERT( + aws_cbor_decoder_pop_next_negative_int_val(decoder, &cbor_data.neg_val) == AWS_OP_SUCCESS); + break; + case AWS_CBOR_TYPE_FLOAT: + AWS_FATAL_ASSERT(aws_cbor_decoder_pop_next_float_val(decoder, &cbor_data.double_val) == AWS_OP_SUCCESS); + break; + case AWS_CBOR_TYPE_BYTES: + AWS_FATAL_ASSERT(aws_cbor_decoder_pop_next_bytes_val(decoder, &cbor_data.bytes_val) == AWS_OP_SUCCESS); + break; + case AWS_CBOR_TYPE_TEXT: + AWS_FATAL_ASSERT(aws_cbor_decoder_pop_next_text_val(decoder, &cbor_data.str_val) == AWS_OP_SUCCESS); + break; + case AWS_CBOR_TYPE_ARRAY_START: + AWS_FATAL_ASSERT( + aws_cbor_decoder_pop_next_array_start(decoder, &cbor_data.array_start) == AWS_OP_SUCCESS); + break; + case AWS_CBOR_TYPE_MAP_START: + AWS_FATAL_ASSERT(aws_cbor_decoder_pop_next_map_start(decoder, &cbor_data.map_start) == AWS_OP_SUCCESS); + break; + case AWS_CBOR_TYPE_TAG: + AWS_FATAL_ASSERT(aws_cbor_decoder_pop_next_tag_val(decoder, &cbor_data.tag_val) == AWS_OP_SUCCESS); + break; + case AWS_CBOR_TYPE_BOOL: + AWS_FATAL_ASSERT( + aws_cbor_decoder_pop_next_boolean_val(decoder, &cbor_data.boolean_val) == AWS_OP_SUCCESS); + break; + case AWS_CBOR_TYPE_NULL: + case AWS_CBOR_TYPE_UNDEFINED: + case AWS_CBOR_TYPE_BREAK: + case AWS_CBOR_TYPE_INDEF_BYTES_START: + case AWS_CBOR_TYPE_INDEF_TEXT_START: + case AWS_CBOR_TYPE_INDEF_ARRAY_START: + case AWS_CBOR_TYPE_INDEF_MAP_START: { + enum aws_cbor_type type = AWS_CBOR_TYPE_UNKNOWN; + AWS_FATAL_ASSERT(aws_cbor_decoder_peek_type(decoder, &type) == AWS_OP_SUCCESS); + AWS_FATAL_ASSERT(type == out_type); + AWS_FATAL_ASSERT(aws_cbor_decoder_consume_next_single_element(decoder) == AWS_OP_SUCCESS); + } break; + + default: + break; + } + } + aws_cbor_decoder_destroy(decoder); + + atexit(aws_common_library_clean_up); + + /* Check for leaks */ + AWS_FATAL_ASSERT(aws_mem_tracer_count(allocator) == 0); + allocator = aws_mem_tracer_destroy(allocator); + return 0; +} diff --git a/tests/fuzz/cbor_double_encode_decode.c b/tests/fuzz/cbor_double_encode_decode.c new file mode 100644 index 000000000..744d1cff8 --- /dev/null +++ b/tests/fuzz/cbor_double_encode_decode.c @@ -0,0 +1,66 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include + +/* NOLINTNEXTLINE(readability-identifier-naming) */ +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + struct aws_allocator *allocator = aws_mem_tracer_new(aws_default_allocator(), NULL, AWS_MEMTRACE_BYTES, 0); + struct aws_byte_cursor input = aws_byte_cursor_from_array(data, size); + double val = 0; + if (!aws_byte_cursor_read_float_be64(&input, &val)) { + allocator = aws_mem_tracer_destroy(allocator); + /* Ignore the invalid double */ + return 0; + } + aws_common_library_init(allocator); + struct aws_cbor_encoder *encoder = aws_cbor_encoder_new(allocator); + aws_cbor_encoder_write_float(encoder, val); + + struct aws_byte_cursor final_cursor = aws_cbor_encoder_get_encoded_data(encoder); + struct aws_cbor_decoder *decoder = aws_cbor_decoder_new(allocator, final_cursor); + enum aws_cbor_type out_type = AWS_CBOR_TYPE_UNKNOWN; + AWS_FATAL_ASSERT(aws_cbor_decoder_peek_type(decoder, &out_type) == 0); + switch (out_type) { + case AWS_CBOR_TYPE_UINT: { + uint64_t result = 0; + AWS_FATAL_ASSERT(aws_cbor_decoder_pop_next_unsigned_int_val(decoder, &result) == 0); + AWS_FATAL_ASSERT((double)result == val); + break; + } + case AWS_CBOR_TYPE_NEGINT: { + uint64_t result = 0; + AWS_FATAL_ASSERT(aws_cbor_decoder_pop_next_negative_int_val(decoder, &result) == 0); + int64_t expected_val = -1 - result; + AWS_FATAL_ASSERT(expected_val == (int64_t)val); + break; + } + case AWS_CBOR_TYPE_FLOAT: { + double result = 0; + AWS_FATAL_ASSERT(aws_cbor_decoder_pop_next_float_val(decoder, &result) == 0); + if (isnan(val)) { + AWS_FATAL_ASSERT(isnan(result)); + } else { + AWS_FATAL_ASSERT(result == val); + } + break; + } + + default: + AWS_FATAL_ASSERT(false); + break; + } + AWS_FATAL_ASSERT(aws_cbor_decoder_get_remaining_length(decoder) == 0); + aws_cbor_encoder_destroy(encoder); + aws_cbor_decoder_destroy(decoder); + + atexit(aws_common_library_clean_up); + + /* Check for leaks */ + AWS_FATAL_ASSERT(aws_mem_tracer_count(allocator) == 0); + allocator = aws_mem_tracer_destroy(allocator); + return 0; +} From fee7aa6800e3f501d80284b723a41160fbfbb396 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 10 Jul 2024 14:28:35 -0700 Subject: [PATCH 35/42] fix FreeBSD CI to install default python packages (#1133) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eff75483f..1919e2d48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,7 +220,7 @@ jobs: operating_system: freebsd version: '13.2' run: | - sudo pkg install -y python3 py39-urllib3 py39-pip cmake + sudo pkg install -y python3 devel/py-pip net/py-urllib3 cmake python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" chmod a+x builder ./builder build -p ${{ env.PACKAGE_NAME }} From 532820ea493efee053ccd5744c8409d002c6070f Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Mon, 15 Jul 2024 09:59:41 -0700 Subject: [PATCH 36/42] Fix out variable in cmake sanitizer module (#1134) --- cmake/AwsSanitizers.cmake | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmake/AwsSanitizers.cmake b/cmake/AwsSanitizers.cmake index c078575c1..54202006e 100644 --- a/cmake/AwsSanitizers.cmake +++ b/cmake/AwsSanitizers.cmake @@ -11,13 +11,15 @@ set(SANITIZERS "address;undefined" CACHE STRING "List of sanitizers to build wit # sanitizer: The sanitizer to check # out_variable: The variable to assign the result to. Defaults to HAS_SANITIZER_${sanitizer} function(aws_check_sanitizer sanitizer) - - if(NOT ${ARGN}) - set(out_variable "${ARGN}") - else() + list(LENGTH ARGN extra_count) + if(${extra_count} EQUAL 0) set(out_variable HAS_SANITIZER_${sanitizer}) # Sanitize the variable name to remove illegal characters string(MAKE_C_IDENTIFIER ${out_variable} out_variable) + elseif(${extra_count} EQUAL 1) + set(out_variable ${ARGN}) + else() + message(FATAL_ERROR "Error: aws_check_sanitizer() called with multiple out variables") endif() if(ENABLE_SANITIZERS) From f4023551346bd9e6f63e0c9811906a7f32453132 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 17 Jul 2024 10:47:39 -0700 Subject: [PATCH 37/42] Update MacOS CI to Arm64 (#1136) --- .github/workflows/ci.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1919e2d48..daaf93d56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,8 +174,8 @@ jobs: echo "Starting to run AppVerifier on all tests found by CTest" python .\aws-c-common\scripts\appverifier_ctest.py --build_directory .\aws-c-common\build\aws-c-common - osx: - runs-on: macos-12 # latest + macos-x64: + runs-on: macos-14-large # latest steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | @@ -183,8 +183,17 @@ jobs: chmod a+x builder ./builder build -p ${{ env.PACKAGE_NAME }} - osx-no-cpu-extensions: - runs-on: macos-12 # latest + macos: + runs-on: macos-14 # latest + steps: + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + macos-no-cpu-extensions: + runs-on: macos-14 # latest steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | @@ -255,8 +264,8 @@ jobs: python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} --config Debug - osx-debug: - runs-on: macos-12 # latest + macos-debug: + runs-on: macos-14 # latest steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | From 852f8ce17c739cc7c1b5585345ddcd913f78ecc8 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 17 Jul 2024 15:00:01 -0700 Subject: [PATCH 38/42] Update builder to fix macos arm64 CI (#1137) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daaf93d56..0f8b49cca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: - 'main' env: - BUILDER_VERSION: v0.9.55 + BUILDER_VERSION: v0.9.62 BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net BUILDER_SOURCE: releases PACKAGE_NAME: aws-c-common From 67601bbbce6ea1c26191f235afc50007a4e796c5 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 24 Jul 2024 10:17:15 -0700 Subject: [PATCH 39/42] Bump the minimum stack size to at least 1MB (#1139) --- source/posix/thread.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/source/posix/thread.c b/source/posix/thread.c index af7fac84c..34b5dbe94 100644 --- a/source/posix/thread.c +++ b/source/posix/thread.c @@ -275,6 +275,25 @@ int aws_thread_launch( if (attr_return) { goto cleanup; } + } else if (!options->stack_size) { + /** + * On some systems, the default stack size is too low (128KB on musl at the time of writing this), which can + * cause stack overflow when the dependency chain is long. Increase the stack size to at + * least 1MB, which is the default on Windows. + */ + size_t min_stack_size = (size_t)1 * 1024 * 1024; + size_t current_stack_size; + attr_return = pthread_attr_getstacksize(attributes_ptr, ¤t_stack_size); + if (attr_return) { + goto cleanup; + } + + if (current_stack_size < min_stack_size) { + attr_return = pthread_attr_setstacksize(attributes_ptr, min_stack_size); + if (attr_return) { + goto cleanup; + } + } } /* AFAIK you can't set thread affinity on apple platforms, and it doesn't really matter since all memory From 622853ccc415874f60c67f62b8108556bd8e2273 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Wed, 24 Jul 2024 11:48:22 -0700 Subject: [PATCH 40/42] Run proofs with CBMC 6.1.0 (#1140) --- .github/workflows/proof_ci_resources/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/proof_ci_resources/config.yaml b/.github/workflows/proof_ci_resources/config.yaml index 4a362f2db..83407e8d9 100644 --- a/.github/workflows/proof_ci_resources/config.yaml +++ b/.github/workflows/proof_ci_resources/config.yaml @@ -1,7 +1,7 @@ # Use exact versions (instead of "latest") so we're not broken by surprise upgrades. cadical-tag: "rel-2.0.0" # tag of latest release: https://github.com/arminbiere/cadical/releases -cbmc-version: "6.0.0" # semver of latest release: https://github.com/diffblue/cbmc/releases -cbmc-viewer-version: "3.8" # semver of latest release: https://github.com/model-checking/cbmc-viewer/releases +cbmc-version: "6.1.0" # semver of latest release: https://github.com/diffblue/cbmc/releases +cbmc-viewer-version: "3.9" # semver of latest release: https://github.com/model-checking/cbmc-viewer/releases kissat-tag: "rel-3.1.1" # tag of latest release: https://github.com/arminbiere/kissat/releases litani-version: "1.29.0" # semver of latest release: https://github.com/awslabs/aws-build-accumulator/releases proofs-dir: verification/cbmc/proofs From c9ead7594928798964e8c85bd14eab71d01389d5 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 30 Jul 2024 01:29:41 +0300 Subject: [PATCH 41/42] Avoiding allocating a handle in the Windows RNG. (#1046) --- source/windows/device_random.c | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/source/windows/device_random.c b/source/windows/device_random.c index 6cb92d43e..0398241dd 100644 --- a/source/windows/device_random.c +++ b/source/windows/device_random.c @@ -5,26 +5,11 @@ #include #include -#include #include #include -static BCRYPT_ALG_HANDLE s_alg_handle = NULL; -static aws_thread_once s_rand_init = AWS_THREAD_ONCE_STATIC_INIT; - -static void s_init_rand(void *user_data) { - (void)user_data; - NTSTATUS status = 0; - - status = BCryptOpenAlgorithmProvider(&s_alg_handle, BCRYPT_RNG_ALGORITHM, NULL, 0); - - if (!BCRYPT_SUCCESS(status)) { - abort(); - } -} - int aws_device_random_buffer(struct aws_byte_buf *output) { return aws_device_random_buffer_append(output, output->capacity - output->len); } @@ -32,8 +17,6 @@ int aws_device_random_buffer(struct aws_byte_buf *output) { int aws_device_random_buffer_append(struct aws_byte_buf *output, size_t n) { AWS_PRECONDITION(aws_byte_buf_is_valid(output)); - aws_thread_call_once(&s_rand_init, s_init_rand, NULL); - size_t space_available = output->capacity - output->len; if (space_available < n) { AWS_POSTCONDITION(aws_byte_buf_is_valid(output)); @@ -47,7 +30,8 @@ int aws_device_random_buffer_append(struct aws_byte_buf *output, size_t n) { while (n > 0) { uint32_t capped_n = (uint32_t)aws_min_size(n, UINT32_MAX); - NTSTATUS status = BCryptGenRandom(s_alg_handle, output->buffer + output->len, capped_n, 0 /*flags*/); + NTSTATUS status = + BCryptGenRandom(NULL, output->buffer + output->len, capped_n, BCRYPT_USE_SYSTEM_PREFERRED_RNG); if (!BCRYPT_SUCCESS(status)) { output->len = original_len; From 0a98aa0b44dfb6c0bc07d9fa6811135c9a579eb9 Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Tue, 30 Jul 2024 08:32:46 -0700 Subject: [PATCH 42/42] Add no copy api variants to json interface (#1138) --- include/aws/common/json.h | 69 ++++++++++++++++++++++++++++- source/json.c | 91 +++++++++++++++++++++------------------ tests/json_test.c | 19 +++++--- 3 files changed, 130 insertions(+), 49 deletions(-) diff --git a/include/aws/common/json.h b/include/aws/common/json.h index b8c4e6cfe..cfda4cf18 100644 --- a/include/aws/common/json.h +++ b/include/aws/common/json.h @@ -23,13 +23,26 @@ AWS_EXTERN_C_BEGIN * * Note: You will need to free the memory for the aws_json_value using aws_json_destroy on the aws_json_value or * on the object/array containing the aws_json_value. - * @param string A byte pointer to the string you want to store in the aws_json_value + * Note: might be slower than c_str version due to internal copy + * @param string A byte cursor you want to store in the aws_json_value * @param allocator The allocator to use when creating the value * @return A new string aws_json_value */ AWS_COMMON_API struct aws_json_value *aws_json_value_new_string(struct aws_allocator *allocator, struct aws_byte_cursor string); +/** + * Creates a new string aws_json_value with the given string and returns a pointer to it. + * + * Note: You will need to free the memory for the aws_json_value using aws_json_destroy on the aws_json_value or + * on the object/array containing the aws_json_value. + * @param string c string pointer you want to store in the aws_json_value + * @param allocator The allocator to use when creating the value + * @return A new string aws_json_value + */ +AWS_COMMON_API +struct aws_json_value *aws_json_value_new_string_from_c_str(struct aws_allocator *allocator, const char *string); + /** * Creates a new number aws_json_value with the given number and returns a pointer to it. * @@ -129,6 +142,7 @@ int aws_json_value_get_boolean(const struct aws_json_value *value, bool *output) * * Note that the aws_json_value will be destroyed when the aws_json_value object is destroyed * by calling "aws_json_destroy()" + * Note: might be slower than c_str version due to internal copy * @param object The object aws_json_value you want to add a value to. * @param key The key to add the aws_json_value at. * @param value The aws_json_value you want to add. @@ -142,8 +156,24 @@ int aws_json_value_add_to_object( struct aws_byte_cursor key, struct aws_json_value *value); +/** + * Adds a aws_json_value to a object aws_json_value. + * + * Note that the aws_json_value will be destroyed when the aws_json_value object is destroyed + * by calling "aws_json_destroy()" + * @param object The object aws_json_value you want to add a value to. + * @param key The key to add the aws_json_value at. + * @param value The aws_json_value you want to add. + * @return AWS_OP_SUCCESS if adding was successful. + * Will return AWS_OP_ERROR if the object passed is invalid or if the passed key + * is already in use in the object. + */ +AWS_COMMON_API +int aws_json_value_add_to_object_c_str(struct aws_json_value *object, const char *key, struct aws_json_value *value); + /** * Returns the aws_json_value at the given key. + * Note: might be slower than c_str version due to internal copy * @param object The object aws_json_value you want to get the value from. * @param key The key that the aws_json_value is at. Is case sensitive. * @return The aws_json_value at the given key, otherwise NULL. @@ -151,8 +181,20 @@ int aws_json_value_add_to_object( AWS_COMMON_API struct aws_json_value *aws_json_value_get_from_object(const struct aws_json_value *object, struct aws_byte_cursor key); +/** + * Returns the aws_json_value at the given key. + * Note: same as aws_json_value_get_from_object but with key as const char *. + * Prefer this method is you have a key thats already a valid char * as it is likely to be faster. + * @param object The object aws_json_value you want to get the value from. + * @param key The key that the aws_json_value is at. Is case sensitive. + * @return The aws_json_value at the given key, otherwise NULL. + */ +AWS_COMMON_API +struct aws_json_value *aws_json_value_get_from_object_c_str(const struct aws_json_value *object, const char *key); + /** * Checks if there is a aws_json_value at the given key. + * Note: might be slower than c_str version due to internal copy * @param object The value aws_json_value you want to check a key in. * @param key The key that you want to check. Is case sensitive. * @return True if a aws_json_value is found. @@ -160,8 +202,20 @@ struct aws_json_value *aws_json_value_get_from_object(const struct aws_json_valu AWS_COMMON_API bool aws_json_value_has_key(const struct aws_json_value *object, struct aws_byte_cursor key); +/** + * Checks if there is a aws_json_value at the given key. + * Note: same as aws_json_value_has_key but with key as const char *. + * Prefer this method is you have a key thats already a valid char * as it is likely to be faster. + * @param object The value aws_json_value you want to check a key in. + * @param key The key that you want to check. Is case sensitive. + * @return True if a aws_json_value is found. + */ +AWS_COMMON_API +bool aws_json_value_has_key_c_str(const struct aws_json_value *object, const char *key); + /** * Removes the aws_json_value at the given key. + * Note: might be slower than c_str version due to internal copy * @param object The object aws_json_value you want to remove a aws_json_value in. * @param key The key that the aws_json_value is at. Is case sensitive. * @return AWS_OP_SUCCESS if the aws_json_value was removed. @@ -171,6 +225,19 @@ bool aws_json_value_has_key(const struct aws_json_value *object, struct aws_byte AWS_COMMON_API int aws_json_value_remove_from_object(struct aws_json_value *object, struct aws_byte_cursor key); +/** + * Removes the aws_json_value at the given key. + * Note: same as aws_json_value_remove_from_object but with key as const char *. + * Prefer this method is you have a key thats already a valid char * as it is likely to be faster. + * @param object The object aws_json_value you want to remove a aws_json_value in. + * @param key The key that the aws_json_value is at. Is case sensitive. + * @return AWS_OP_SUCCESS if the aws_json_value was removed. + * Will return AWS_OP_ERR if the object passed is invalid or if the value + * at the key cannot be found. + */ +AWS_COMMON_API +int aws_json_value_remove_from_object_c_str(struct aws_json_value *object, const char *key); + /** * @brief callback for iterating members of an object * Iteration can be controlled as follows: diff --git a/source/json.c b/source/json.c index 28524a88c..2f1630ea2 100644 --- a/source/json.c +++ b/source/json.c @@ -21,6 +21,12 @@ struct aws_json_value *aws_json_value_new_string(struct aws_allocator *allocator return ret_val; } +struct aws_json_value *aws_json_value_new_string_from_c_str(struct aws_allocator *allocator, const char *string) { + (void)allocator; /* No need for allocator. It is overriden through hooks. */ + void *ret_val = cJSON_CreateString(string); + return ret_val; +} + struct aws_json_value *aws_json_value_new_number(struct aws_allocator *allocator, double number) { (void)allocator; // prevent warnings over unused parameter return (void *)cJSON_CreateNumber(number); @@ -78,92 +84,95 @@ int aws_json_value_add_to_object( struct aws_byte_cursor key, struct aws_json_value *value) { - int result = AWS_OP_ERR; struct aws_string *tmp = aws_string_new_from_cursor(s_aws_json_module_allocator, &key); + int result = aws_json_value_add_to_object_c_str(object, aws_string_c_str(tmp), value); + + aws_string_destroy_secure(tmp); + return result; +} + +int aws_json_value_add_to_object_c_str(struct aws_json_value *object, const char *key, struct aws_json_value *value) { struct cJSON *cjson = (struct cJSON *)object; if (!cJSON_IsObject(cjson)) { - aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - goto done; + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } struct cJSON *cjson_value = (struct cJSON *)value; if (cJSON_IsInvalid(cjson_value)) { - result = aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - goto done; + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } - if (cJSON_HasObjectItem(cjson, aws_string_c_str(tmp))) { - goto done; + if (cJSON_HasObjectItem(cjson, key)) { + return AWS_OP_ERR; } - cJSON_AddItemToObject(cjson, aws_string_c_str(tmp), cjson_value); - result = AWS_OP_SUCCESS; - -done: - aws_string_destroy_secure(tmp); - return result; + cJSON_AddItemToObject(cjson, key, cjson_value); + return AWS_OP_SUCCESS; } struct aws_json_value *aws_json_value_get_from_object(const struct aws_json_value *object, struct aws_byte_cursor key) { - void *return_value = NULL; struct aws_string *tmp = aws_string_new_from_cursor(s_aws_json_module_allocator, &key); + void *return_value = aws_json_value_get_from_object_c_str(object, aws_string_c_str(tmp)); + aws_string_destroy_secure(tmp); + return return_value; +} + +struct aws_json_value *aws_json_value_get_from_object_c_str(const struct aws_json_value *object, const char *key) { const struct cJSON *cjson = (const struct cJSON *)object; if (!cJSON_IsObject(cjson)) { aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - goto done; + return NULL; } - if (!cJSON_HasObjectItem(cjson, aws_string_c_str(tmp))) { - goto done; + if (!cJSON_HasObjectItem(cjson, key)) { + return NULL; } - return_value = (void *)cJSON_GetObjectItem(cjson, aws_string_c_str(tmp)); - -done: - aws_string_destroy_secure(tmp); - return return_value; + return (void *)cJSON_GetObjectItem(cjson, key); } bool aws_json_value_has_key(const struct aws_json_value *object, struct aws_byte_cursor key) { struct aws_string *tmp = aws_string_new_from_cursor(s_aws_json_module_allocator, &key); - bool result = false; + bool result = aws_json_value_has_key_c_str(object, aws_string_c_str(tmp)); + + aws_string_destroy_secure(tmp); + return result; +} +bool aws_json_value_has_key_c_str(const struct aws_json_value *object, const char *key) { const struct cJSON *cjson = (const struct cJSON *)object; if (!cJSON_IsObject(cjson)) { - goto done; + return false; } - if (!cJSON_HasObjectItem(cjson, aws_string_c_str(tmp))) { - goto done; + if (!cJSON_HasObjectItem(cjson, key)) { + return false; } - result = true; -done: - aws_string_destroy_secure(tmp); - return result; + return true; } int aws_json_value_remove_from_object(struct aws_json_value *object, struct aws_byte_cursor key) { - int result = AWS_OP_ERR; struct aws_string *tmp = aws_string_new_from_cursor(s_aws_json_module_allocator, &key); + int result = aws_json_value_remove_from_object_c_str(object, aws_string_c_str(tmp)); + aws_string_destroy_secure(tmp); + return result; +} + +int aws_json_value_remove_from_object_c_str(struct aws_json_value *object, const char *key) { struct cJSON *cjson = (struct cJSON *)object; if (!cJSON_IsObject(cjson)) { - aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - goto done; + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } - if (!cJSON_HasObjectItem(cjson, aws_string_c_str(tmp))) { - goto done; + if (!cJSON_HasObjectItem(cjson, key)) { + return AWS_OP_ERR; } - cJSON_DeleteItemFromObject(cjson, aws_string_c_str(tmp)); - result = AWS_OP_SUCCESS; - -done: - aws_string_destroy_secure(tmp); - return result; + cJSON_DeleteItemFromObject(cjson, key); + return AWS_OP_SUCCESS; } int aws_json_const_iterate_object( diff --git a/tests/json_test.c b/tests/json_test.c index 8ba2a1c07..ed10fbee6 100644 --- a/tests/json_test.c +++ b/tests/json_test.c @@ -114,6 +114,7 @@ static int s_test_json_parse_from_string(struct aws_allocator *allocator, void * // Testing valid array struct aws_json_value *array_node = aws_json_value_get_from_object(root, aws_byte_cursor_from_c_str("array")); + ASSERT_PTR_EQUALS(array_node, aws_json_value_get_from_object_c_str(root, "array")); ASSERT_NOT_NULL(array_node); ASSERT_TRUE(aws_json_value_is_array(array_node)); ASSERT_TRUE(aws_json_get_array_size(array_node) == 3); @@ -163,12 +164,14 @@ static int s_test_json_parse_from_string(struct aws_allocator *allocator, void * aws_string_destroy_secure(tmp_str); // Testing valid number - struct aws_json_value *number_node = aws_json_value_get_from_object(root, aws_byte_cursor_from_c_str("number")); + struct aws_json_value *number_node = aws_json_value_get_from_object_c_str(root, "number"); ASSERT_NOT_NULL(number_node); ASSERT_TRUE(aws_json_value_is_number(number_node)); double double_test_two = 0; aws_json_value_get_number(number_node, &double_test_two); ASSERT_TRUE(double_test_two == (double)123); + ASSERT_TRUE(aws_json_value_has_key_c_str(root, "number")); + ASSERT_TRUE(aws_json_value_has_key(root, aws_byte_cursor_from_c_str("number"))); // Testing valid object struct aws_json_value *object_node = aws_json_value_get_from_object(root, aws_byte_cursor_from_c_str("object")); @@ -210,6 +213,12 @@ static int s_test_json_parse_from_string(struct aws_allocator *allocator, void * // Test getting invalid type of data ASSERT_INT_EQUALS(aws_json_value_get_number(string_node, NULL), AWS_OP_ERR); + ASSERT_SUCCESS(aws_json_value_remove_from_object(root, aws_byte_cursor_from_c_str("number"))); + ASSERT_FALSE(aws_json_value_has_key_c_str(root, "number")); + + ASSERT_SUCCESS(aws_json_value_remove_from_object_c_str(root, "object")); + ASSERT_FALSE(aws_json_value_has_key_c_str(root, "object")); + aws_json_value_destroy(root); // Make sure that destroying NULL does not have any bad effects. @@ -233,12 +242,8 @@ static int s_test_json_parse_to_string(struct aws_allocator *allocator, void *ct aws_json_value_add_array_element(array, aws_json_value_new_number(allocator, 3)); aws_json_value_add_to_object(root, aws_byte_cursor_from_c_str("array"), array); - aws_json_value_add_to_object( - root, aws_byte_cursor_from_c_str("boolean"), aws_json_value_new_boolean(allocator, true)); - aws_json_value_add_to_object( - root, - aws_byte_cursor_from_c_str("color"), - aws_json_value_new_string(allocator, aws_byte_cursor_from_c_str("gold"))); + aws_json_value_add_to_object_c_str(root, "boolean", aws_json_value_new_boolean(allocator, true)); + aws_json_value_add_to_object_c_str(root, "color", aws_json_value_new_string_from_c_str(allocator, "gold")); aws_json_value_add_to_object(root, aws_byte_cursor_from_c_str("null"), aws_json_value_new_null(allocator)); aws_json_value_add_to_object(root, aws_byte_cursor_from_c_str("number"), aws_json_value_new_number(allocator, 123));