From ad6be8e51e28f6d3d78e3814e539684999bd7702 Mon Sep 17 00:00:00 2001 From: Mark Gillard Date: Sat, 8 Aug 2020 20:51:33 +0300 Subject: [PATCH] minor cleanup mostly for testing on ICC. --- include/toml++/toml_node.h | 2 +- include/toml++/toml_parser.h | 2 +- include/toml++/toml_utf8.h | 6 +- include/toml++/toml_utf8_streams.h | 4 +- meson.build | 19 ++++++- python/generate_conformance_tests.py | 28 ++++++++- tests/conformance_burntsushi_valid.cpp | 6 ++ tests/conformance_iarna_valid.cpp | 10 ++++ tests/meson.build | 79 +++++++++++++++++++++++--- tests/parsing_key_value_pairs.cpp | 4 ++ tests/parsing_strings.cpp | 4 ++ tests/parsing_tables.cpp | 6 +- tests/tests.h | 6 ++ toml.hpp | 13 +++-- 14 files changed, 165 insertions(+), 24 deletions(-) diff --git a/include/toml++/toml_node.h b/include/toml++/toml_node.h index 8808ac2..c0f8acc 100644 --- a/include/toml++/toml_node.h +++ b/include/toml++/toml_node.h @@ -10,7 +10,7 @@ TOML_PUSH_WARNINGS TOML_DISABLE_PADDING_WARNINGS TOML_DISABLE_MISC_WARNINGS -#if defined(doxygen) || TOML_SIMPLE_STATIC_ASSERT_MESSAGES +#if defined(DOXYGEN) || TOML_SIMPLE_STATIC_ASSERT_MESSAGES #define TOML_SA_NEWLINE " " #define TOML_SA_LIST_SEP ", " diff --git a/include/toml++/toml_parser.h b/include/toml++/toml_parser.h index 0d660e8..f7f3416 100644 --- a/include/toml++/toml_parser.h +++ b/include/toml++/toml_parser.h @@ -686,7 +686,7 @@ TOML_NAMESPACE_START const auto file_size = file.tellg(); if (file_size == -1) TOML_THROW_PARSE_ERROR("Could not determine file size", file_path_str); - file.seekg(0, std::ios::beg); + file.seekg(0, ifstream::beg); // read the whole file into memory first if the file isn't too large constexpr auto large_file_threshold = 1024 * 1024 * static_cast(sizeof(void*)) * 4; // 32 megabytes on 64-bit diff --git a/include/toml++/toml_utf8.h b/include/toml++/toml_utf8.h index 33508ad..cc3be7f 100644 --- a/include/toml++/toml_utf8.h +++ b/include/toml++/toml_utf8.h @@ -1,6 +1,5 @@ //# This file is a part of toml++ and is subject to the the terms of the MIT license. //# Copyright (c) 2019-2020 Mark Gillard -//# Copyright (c) 2008-2010 Bjoern Hoehrmann (utf8_decoder) //# See https://github.com/marzer/tomlplusplus/blob/master/LICENSE for the full license text. // SPDX-License-Identifier: MIT @@ -909,10 +908,11 @@ TOML_IMPL_NAMESPACE_START return codepoint >= 0xD800u && codepoint <= 0xDFFF; } - // utf8_decoder based on this: https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - // Copyright (c) 2008-2009 Bjoern Hoehrmann struct utf8_decoder final { + // utf8_decoder based on this: https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + // Copyright (c) 2008-2009 Bjoern Hoehrmann + uint_least32_t state{}; char32_t codepoint{}; diff --git a/include/toml++/toml_utf8_streams.h b/include/toml++/toml_utf8_streams.h index d0be52d..e55df38 100644 --- a/include/toml++/toml_utf8_streams.h +++ b/include/toml++/toml_utf8_streams.h @@ -107,7 +107,7 @@ TOML_IMPL_NAMESPACE_START return; source->clear(); - source->seekg(initial_pos, std::ios::beg); + source->seekg(initial_pos, std::basic_istream::beg); } [[nodiscard]] @@ -320,7 +320,9 @@ TOML_IMPL_NAMESPACE_START } } + #if !TOML_ICC TOML_UNREACHABLE; + #endif } [[nodiscard]] diff --git a/meson.build b/meson.build index 994c179..03bd0eb 100644 --- a/meson.build +++ b/meson.build @@ -25,6 +25,15 @@ if build_tests or build_examples compiler = meson.get_compiler('cpp') message(['compiler ID: ', compiler.get_id()]) message(['compiler version: ', compiler.version()]) + + # intel-cl __builtin_bit_cast workaround + if compiler.get_id() == 'intel-cl' + add_project_arguments([ + '/D__builtin_bit_cast(x,y)=x{}' + ], + language : 'cpp' + ) + endif # GCC or Clang if compiler.get_id() == 'gcc' or compiler.get_id() == 'clang' @@ -124,11 +133,19 @@ if build_tests or build_examples endif endif + # msvc + if compiler.get_id() == 'msvc' + add_project_arguments([ + '/utf-8' + ], + language : 'cpp' + ) + endif + # ICC's visual studio frontend, ICL if compiler.get_id() == 'intel-cl' add_project_arguments([ '/Qoption,cpp,--unicode_source_kind,UTF-8', - '/std=c++latest', '/wd82', # storage class is not first '/wd280', # selector expression is constant (why the fuck is that a warning?) '/wd411', # class provides no constructor (duh, it's an aggregate) diff --git a/python/generate_conformance_tests.py b/python/generate_conformance_tests.py index c0e2c97..40269d8 100644 --- a/python/generate_conformance_tests.py +++ b/python/generate_conformance_tests.py @@ -217,6 +217,7 @@ class TomlTest: self.__identifier = sanitize(name) self.__data = utils.read_all_text_from_file(file_path).strip() self.condition = '' + self.requires_unicode = False if is_valid_case: self.__expected = True path_base = path.splitext(file_path)[0] @@ -358,6 +359,15 @@ def load_invalid_inputs(tests, extern_root): )) + +def requires_unicode(s): + for c in s: + if ord(c) > 127: + return True + return False + + + def write_test_file(name, test_cases): conditions = set() @@ -388,7 +398,14 @@ def write_test_file(name, test_cases): write('namespace') write('{') for test in test_cases: - write('\t{}'.format(test)) + s = '\t{}'.format(test) + test.requires_unicode = requires_unicode(s) + if test.requires_unicode: + write('\t#if UNICODE_LITERALS_OK') + write(s) + write('\t#endif // UNICODE_LITERALS_OK') + else: + write(s) write('}') write('') write('TOML_POP_WARNINGS') @@ -411,11 +428,18 @@ def write_test_file(name, test_cases): else: write('\tparsing_should_fail(FILE_LINE_ARGS, {});'.format(test.identifier())) else: + s = expected.render('\t\t') + if not test.requires_unicode: + test.requires_unicode = requires_unicode(s) + if test.requires_unicode: + write('\t#if UNICODE_LITERALS_OK') write('\tparsing_should_succeed(FILE_LINE_ARGS, {}, [](toml::table&& tbl)'.format(test.identifier())) write('\t{') - write('\t\tauto expected = {};'.format(expected.render('\t\t'))) + write('\t\tauto expected = {};'.format(s)) write('\t\tREQUIRE(tbl == expected);') write('\t});') + if test.requires_unicode: + write('\t#endif // UNICODE_LITERALS_OK') write('') if condition != '': write('\t#endif // {}'.format(condition)); diff --git a/tests/conformance_burntsushi_valid.cpp b/tests/conformance_burntsushi_valid.cpp index c918731..a21ef1d 100644 --- a/tests/conformance_burntsushi_valid.cpp +++ b/tests/conformance_burntsushi_valid.cpp @@ -252,7 +252,9 @@ answer = 42)"sv; static constexpr auto underscored_integer = R"(million = 1_000_000)"sv; static constexpr auto unicode_escape = R"(answer4 = "\u03B4" answer8 = "\U000003B4")"sv; + #if UNICODE_LITERALS_OK static constexpr auto unicode_literal = R"(answer = "δ")"sv; + #endif // UNICODE_LITERALS_OK } TOML_POP_WARNINGS @@ -1135,6 +1137,7 @@ ue)"sv }, REQUIRE(tbl == expected); }); + #if UNICODE_LITERALS_OK parsing_should_succeed(FILE_LINE_ARGS, unicode_escape, [](toml::table&& tbl) { auto expected = toml::table{{ @@ -1143,7 +1146,9 @@ ue)"sv }, }}; REQUIRE(tbl == expected); }); + #endif // UNICODE_LITERALS_OK + #if UNICODE_LITERALS_OK parsing_should_succeed(FILE_LINE_ARGS, unicode_literal, [](toml::table&& tbl) { auto expected = toml::table{{ @@ -1151,6 +1156,7 @@ ue)"sv }, }}; REQUIRE(tbl == expected); }); + #endif // UNICODE_LITERALS_OK } diff --git a/tests/conformance_iarna_valid.cpp b/tests/conformance_iarna_valid.cpp index de50a2b..8536622 100644 --- a/tests/conformance_iarna_valid.cpp +++ b/tests/conformance_iarna_valid.cpp @@ -157,7 +157,9 @@ orange.color = "orange")"sv; static constexpr auto spec_key_value_pair_7 = R"(_=1)"sv; static constexpr auto spec_key_value_pair_8 = R"(-_-_-_-_-=1)"sv; static constexpr auto spec_key_value_pair_9 = R"(3.14159 = "pi")"sv; + #if UNICODE_LITERALS_OK static constexpr auto spec_quoted_basic_keys_1 = R"("ʎǝʞ" = "value")"sv; + #endif // UNICODE_LITERALS_OK static constexpr auto spec_quoted_literal_keys_1 = R"('quoted "value"' = "value")"sv; static constexpr auto spec_readme_example = R"(# This is a TOML document. @@ -243,7 +245,9 @@ type.name = "pug")"sv; static constexpr auto spec_table_3 = R"([a.b.c])"sv; static constexpr auto spec_table_4 = R"([ d.e.f ] # same as [d.e.f])"sv; static constexpr auto spec_table_5 = R"([ g . h . i ] # same as [g.h.i])"sv; + #if UNICODE_LITERALS_OK static constexpr auto spec_table_6 = R"([ j . "ʞ" . 'l' ] # same as [j."ʞ".'l'])"sv; + #endif // UNICODE_LITERALS_OK static constexpr auto spec_table_7 = R"(# [x] you # [x.y] don't # [x.y.z] need these @@ -1065,6 +1069,7 @@ TEST_CASE("conformance - iarna/valid") REQUIRE(tbl == expected); }); + #if UNICODE_LITERALS_OK parsing_should_succeed(FILE_LINE_ARGS, spec_quoted_basic_keys_1, [](toml::table&& tbl) { auto expected = toml::table{{ @@ -1072,6 +1077,7 @@ TEST_CASE("conformance - iarna/valid") }}; REQUIRE(tbl == expected); }); + #endif // UNICODE_LITERALS_OK parsing_should_succeed(FILE_LINE_ARGS, spec_quoted_literal_keys_1, [](toml::table&& tbl) { @@ -1229,6 +1235,7 @@ Violets are blue)"sv }, REQUIRE(tbl == expected); }); + #if UNICODE_LITERALS_OK parsing_should_succeed(FILE_LINE_ARGS, spec_string_basic, [](toml::table&& tbl) { auto expected = toml::table{{ @@ -1237,6 +1244,7 @@ Location SF.)"sv }, }}; REQUIRE(tbl == expected); }); + #endif // UNICODE_LITERALS_OK parsing_should_succeed(FILE_LINE_ARGS, spec_string_literal_1, [](toml::table&& tbl) { @@ -1393,6 +1401,7 @@ trimmed in raw strings. REQUIRE(tbl == expected); }); + #if UNICODE_LITERALS_OK parsing_should_succeed(FILE_LINE_ARGS, spec_table_6, [](toml::table&& tbl) { auto expected = toml::table{{ @@ -1408,6 +1417,7 @@ trimmed in raw strings. }}; REQUIRE(tbl == expected); }); + #endif // UNICODE_LITERALS_OK parsing_should_succeed(FILE_LINE_ARGS, spec_table_7, [](toml::table&& tbl) { diff --git a/tests/meson.build b/tests/meson.build index e82734a..f23fc32 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -24,6 +24,24 @@ test_sources = [ 'windows_compat.cpp' ] +# compiler argument references: +# msvc: https://docs.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-alphabetically?view=vs-2019 +# intel and intel-cl: https://software.intel.com/content/www/us/en/develop/documentation/cpp-compiler-oneapi-dev-guide-and-reference/top/compiler-reference/compiler-options/alphabetical-list-of-compiler-options.html +# gcc: +# clang: + +####################################################################################################################### +# c++ 20 check +####################################################################################################################### + +compiler_supports_cpp20_args = [] +if compiler.get_id() == 'gcc' or compiler.get_id() == 'clang' + compiler_supports_cpp20_args += '-std=c++2a' +elif compiler.get_id() == 'intel-cl' + compiler_supports_cpp20_args += '/Qstd=c++2a' +elif compiler.get_id() == 'msvc' + compiler_supports_cpp20_args += '/std=c++latest' +endif compiler_supports_cpp20 = compiler.links(''' #include #include @@ -37,9 +55,18 @@ compiler_supports_cpp20 = compiler.links(''' } ''', name : 'supports c++20', - args : [ '-std=c++2a' ] + args : compiler_supports_cpp20_args ) +####################################################################################################################### +# char8_t check +####################################################################################################################### + +compiler_supports_char8_args = [] +compiler_supports_char8_args += compiler_supports_cpp20_args +if compiler.get_id() == 'gcc' or compiler.get_id() == 'clang' + compiler_supports_char8_args += '-fchar8_t' +endif compiler_supports_char8 = compiler_supports_cpp20 and compiler.links(''' #include #include @@ -65,9 +92,13 @@ compiler_supports_char8 = compiler_supports_cpp20 and compiler.links(''' } ''', name : 'supports char8_t', - args : [ '-std=c++2a', '-fchar8_t' ] + args : compiler_supports_char8_args ) +####################################################################################################################### +# consteval check +####################################################################################################################### + compiler_supports_consteval = compiler_supports_cpp20 and compiler.compiles(''' consteval int kek() noexcept @@ -81,9 +112,13 @@ compiler_supports_consteval = compiler_supports_cpp20 and compiler.compiles(''' } ''', name : 'supports consteval', - args : [ '-std=c++2a' ] + args : compiler_supports_cpp20_args ) +####################################################################################################################### +# __fp16 and _Float16 checks +####################################################################################################################### + float_16_preprocessor_single_check_template = ''' #ifndef @0@ #error @0@ wasn't defined! @@ -106,7 +141,14 @@ float_16_preprocessor_checks = ''' + float_16_preprocessor_single_check_template.format('__FLT16_MAX_EXP__', '16') \ + float_16_preprocessor_single_check_template.format('__FLT16_MAX_10_EXP__', '4') -compiler_supports_float16_args = [ '-O0' ] +compiler_disable_optimizations = '' +if compiler.get_id() == 'gcc' or compiler.get_id() == 'clang' + compiler_disable_optimizations = '-O0' +elif compiler.get_id() == 'msvc' or compiler.get_id() == 'intel-cl' + compiler_disable_optimizations = '/Od' +endif + +compiler_supports_float16_args = [ compiler_disable_optimizations ] if compiler.get_id() == 'gcc' compiler_supports_float16_args += '-mfp16-format=ieee' compiler_supports_float16_args += '-fmax-errors=1' @@ -144,6 +186,10 @@ compiler_supports_float16 = compiler.links(''' args : compiler_supports_float16_args ) +####################################################################################################################### +# int128 check +####################################################################################################################### + compiler_supports_int128 = compiler.links(''' #ifndef __SIZEOF_INT128__ #error __SIZEOF_INT128__ wasn't defined! @@ -163,9 +209,13 @@ compiler_supports_int128 = compiler.links(''' } ''', name : 'supports __int128_t', - args : [ '-O0' ] + args : [ compiler_disable_optimizations ] ) +####################################################################################################################### +# float128 check +####################################################################################################################### + compiler_supports_float128 = compiler.links(''' #ifndef __SIZEOF_FLOAT128__ #error __SIZEOF_FLOAT128__ wasn't defined! @@ -190,9 +240,20 @@ compiler_supports_float128 = compiler.links(''' } ''', name : 'supports __float128', - args : [ '-O0' ] + args : [ compiler_disable_optimizations ] ) +####################################################################################################################### +# fast math check +####################################################################################################################### + +compiler_supports_fast_math_args = [] +if compiler.get_id() == 'gcc' or compiler.get_id() == 'clang' + compiler_supports_fast_math_args += '-ffast-math' + compiler_supports_fast_math_args += '-ffp-contract=fast' +elif compiler.get_id() == 'msvc' or compiler.get_id() == 'intel-cl' + compiler_supports_fast_math_args += '/fp:fast' +endif compiler_supports_fast_math = compiler.links(''' #include #include @@ -203,9 +264,13 @@ compiler_supports_fast_math = compiler.links(''' } ''', name : 'supports fast-math', - args : [ '-ffast-math', '-ffp-contract=fast' ] + args : compiler_supports_fast_math_args ) +####################################################################################################################### +# do the thing! +####################################################################################################################### + fast_math_modes = [ false, true ] exception_modes = [ true, false ] unreleased_feature_modes = [ false, true ] diff --git a/tests/parsing_key_value_pairs.cpp b/tests/parsing_key_value_pairs.cpp index 44a5c26..c5c7ecd 100644 --- a/tests/parsing_key_value_pairs.cpp +++ b/tests/parsing_key_value_pairs.cpp @@ -29,6 +29,7 @@ TEST_CASE("parsing - key-value pairs") parsing_should_fail(FILE_LINE_ARGS, R"(key = # INVALID)"sv); + #if UNICODE_LITERALS_OK parsing_should_succeed( FILE_LINE_ARGS, R"( @@ -49,6 +50,7 @@ TEST_CASE("parsing - key-value pairs") CHECK(tbl[""] == "blank"sv); } ); + #endif // UNICODE_LITERALS_OK parsing_should_fail(FILE_LINE_ARGS, R"(= "no key name")"sv); @@ -151,6 +153,7 @@ TEST_CASE("parsing - key-value pairs (dotted)") ); // toml/issues/644 ('+' in bare keys) & toml/issues/687 (unicode bare keys) + #if UNICODE_LITERALS_OK #if TOML_LANG_UNRELEASED parsing_should_succeed( FILE_LINE_ARGS, @@ -169,6 +172,7 @@ TEST_CASE("parsing - key-value pairs (dotted)") parsing_should_fail(FILE_LINE_ARGS, R"(key+1 = 0)"sv); parsing_should_fail(FILE_LINE_ARGS, R"(ʎǝʞ2 = 0)"sv); #endif + #endif // UNICODE_LITERALS_OK } TEST_CASE("parsing - key-value pairs (string keys)") diff --git a/tests/parsing_strings.cpp b/tests/parsing_strings.cpp index ce9d884..42338fc 100644 --- a/tests/parsing_strings.cpp +++ b/tests/parsing_strings.cpp @@ -153,6 +153,8 @@ str = ''''That's still pointless', she said.''' FILE_LINE_ARGS, R"('''The quick brown fox jumps over the lazy dog''')"sv, "The quick brown fox jumps over the lazy dog"sv); + + #if UNICODE_LITERALS_OK parse_expected_value( FILE_LINE_ARGS, R"("Ýôú'ℓℓ λáƭè ₥è áƒƭèř ƭλïƨ - #")"sv, @@ -165,6 +167,8 @@ str = ''''That's still pointless', she said.''' FILE_LINE_ARGS, R"("Ýôú δôñ'ƭ ƭλïñƙ ƨô₥è úƨèř ωôñ'ƭ δô ƭλáƭ?")"sv, R"(Ýôú δôñ'ƭ ƭλïñƙ ƨô₥è úƨèř ωôñ'ƭ δô ƭλáƭ?)"sv); + #endif // UNICODE_LITERALS_OK + parse_expected_value( FILE_LINE_ARGS, R"("\"\u03B1\u03B2\u03B3\"")"sv, diff --git a/tests/parsing_tables.cpp b/tests/parsing_tables.cpp index 8db8884..7e51690 100644 --- a/tests/parsing_tables.cpp +++ b/tests/parsing_tables.cpp @@ -26,7 +26,7 @@ type.name = "pug" [a.b.c] # this is best practice [ d.e.f ] # same as [d.e.f] [ g . h . i ] # same as [g.h.i] -[ j . "ʞ" . 'l' ] # same as [j."ʞ".'l'] +[ j . "k" . 'l' ] # same as [j."k".'l'] # [x] you # [x.y] don't @@ -78,8 +78,8 @@ smooth = true CHECK(tbl["g"]["h"]["i"].as()); CHECK(tbl["j"].as
()); - CHECK(tbl["j"]["ʞ"].as
()); - CHECK(tbl["j"]["ʞ"]["l"].as
()); + CHECK(tbl["j"]["k"].as
()); + CHECK(tbl["j"]["k"]["l"].as
()); REQUIRE(tbl["fruit"].as
()); CHECK(tbl["fruit"]["apple"]["color"] == "red"sv); diff --git a/tests/tests.h b/tests/tests.h index 68e6120..76103c5 100644 --- a/tests/tests.h +++ b/tests/tests.h @@ -18,6 +18,12 @@ #include "../include/toml++/toml.h" #endif +#if TOML_ICC + #define UNICODE_LITERALS_OK 0 +#else + #define UNICODE_LITERALS_OK 1 +#endif + TOML_DISABLE_ARITHMETIC_WARNINGS TOML_PUSH_WARNINGS diff --git a/toml.hpp b/toml.hpp index f00519f..0ad59bb 100644 --- a/toml.hpp +++ b/toml.hpp @@ -2031,7 +2031,7 @@ TOML_PUSH_WARNINGS TOML_DISABLE_PADDING_WARNINGS TOML_DISABLE_MISC_WARNINGS -#if defined(doxygen) || TOML_SIMPLE_STATIC_ASSERT_MESSAGES +#if defined(DOXYGEN) || TOML_SIMPLE_STATIC_ASSERT_MESSAGES #define TOML_SA_NEWLINE " " #define TOML_SA_LIST_SEP ", " @@ -5695,10 +5695,11 @@ TOML_IMPL_NAMESPACE_START return codepoint >= 0xD800u && codepoint <= 0xDFFF; } - // utf8_decoder based on this: https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - // Copyright (c) 2008-2009 Bjoern Hoehrmann struct utf8_decoder final { + // utf8_decoder based on this: https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + // Copyright (c) 2008-2009 Bjoern Hoehrmann + uint_least32_t state{}; char32_t codepoint{}; @@ -6713,7 +6714,7 @@ TOML_IMPL_NAMESPACE_START return; source->clear(); - source->seekg(initial_pos, std::ios::beg); + source->seekg(initial_pos, std::basic_istream::beg); } [[nodiscard]] @@ -6925,7 +6926,9 @@ TOML_IMPL_NAMESPACE_START } } + #if !TOML_ICC TOML_UNREACHABLE; + #endif } [[nodiscard]] @@ -7353,7 +7356,7 @@ TOML_NAMESPACE_START const auto file_size = file.tellg(); if (file_size == -1) TOML_THROW_PARSE_ERROR("Could not determine file size", file_path_str); - file.seekg(0, std::ios::beg); + file.seekg(0, ifstream::beg); // read the whole file into memory first if the file isn't too large constexpr auto large_file_threshold = 1024 * 1024 * static_cast(sizeof(void*)) * 4; // 32 megabytes on 64-bit