From ae24b30e05ec90e8b7739817a2a13f32096b8640 Mon Sep 17 00:00:00 2001 From: Yusuke Suzuki Date: Wed, 21 Aug 2024 08:05:56 -0700 Subject: [PATCH] [JSC] JSON name serialization should also take SIMD https://bugs.webkit.org/show_bug.cgi?id=274058 rdar://127970306 Reviewed by Justin Michaud. We should use SIMD code for name serialization too. * Source/JavaScriptCore/runtime/JSONObject.cpp: (JSC::stringCopySameType): (JSC::stringCopyUpconvert): (JSC::FastStringifier::append): Canonical link: https://commits.webkit.org/282560@main --- Source/JavaScriptCore/runtime/JSONObject.cpp | 223 ++++++++++--------- 1 file changed, 115 insertions(+), 108 deletions(-) diff --git a/Source/JavaScriptCore/runtime/JSONObject.cpp b/Source/JavaScriptCore/runtime/JSONObject.cpp index 0c1456f27f889..fcceb0a168473 100644 --- a/Source/JavaScriptCore/runtime/JSONObject.cpp +++ b/Source/JavaScriptCore/runtime/JSONObject.cpp @@ -1044,6 +1044,104 @@ inline void FastStringifier::append(char a, char b, char c m_length += 5; } +template +static ALWAYS_INLINE bool stringCopySameType(std::span span, CharType* cursor) +{ +#if (CPU(ARM64) || CPU(X86_64)) && COMPILER(CLANG) + constexpr size_t stride = SIMD::stride; + if (span.size() >= stride) { + using UnsignedType = std::make_unsigned_t; + using BulkType = decltype(SIMD::load(static_cast(nullptr))); + constexpr auto quoteMask = SIMD::splat('"'); + constexpr auto escapeMask = SIMD::splat('\\'); + constexpr auto controlMask = SIMD::splat(' '); + const auto* ptr = span.data(); + const auto* end = ptr + span.size(); + auto* cursorEnd = cursor + span.size(); + BulkType accumulated { }; + for (; ptr + (stride - 1) < end; ptr += stride, cursor += stride) { + auto input = SIMD::load(bitwise_cast(ptr)); + SIMD::store(input, bitwise_cast(cursor)); + auto quotes = SIMD::equal(input, quoteMask); + auto escapes = SIMD::equal(input, escapeMask); + auto controls = SIMD::lessThan(input, controlMask); + accumulated = SIMD::bitOr(accumulated, quotes, escapes, controls); + if constexpr (sizeof(CharType) != 1) { + constexpr auto surrogateMask = SIMD::splat(0xf800); + constexpr auto surrogateCheckMask = SIMD::splat(0xd800); + accumulated = SIMD::bitOr(accumulated, SIMD::equal(SIMD::bitAnd(input, surrogateMask), surrogateCheckMask)); + } + } + if (ptr < end) { + auto input = SIMD::load(bitwise_cast(end - stride)); + SIMD::store(input, bitwise_cast(cursorEnd - stride)); + auto quotes = SIMD::equal(input, quoteMask); + auto escapes = SIMD::equal(input, escapeMask); + auto controls = SIMD::lessThan(input, controlMask); + accumulated = SIMD::bitOr(accumulated, quotes, escapes, controls); + if constexpr (sizeof(CharType) != 1) { + constexpr auto surrogateMask = SIMD::splat(0xf800); + constexpr auto surrogateCheckMask = SIMD::splat(0xd800); + accumulated = SIMD::bitOr(accumulated, SIMD::equal(SIMD::bitAnd(input, surrogateMask), surrogateCheckMask)); + } + } + return SIMD::isNonZero(accumulated); + } +#endif + for (auto character : span) { + if constexpr (sizeof(CharType) != 1) { + if (UNLIKELY(U16_IS_SURROGATE(character))) + return true; + } + if (UNLIKELY(character <= 0xff && WTF::escapedFormsForJSON[character])) + return true; + *cursor++ = character; + } + return false; +} + +static ALWAYS_INLINE bool stringCopyUpconvert(std::span span, UChar* cursor) +{ +#if (CPU(ARM64) || CPU(X86_64)) && COMPILER(CLANG) + constexpr size_t stride = SIMD::stride; + if (span.size() >= stride) { + using UnsignedType = std::make_unsigned_t; + using BulkType = decltype(SIMD::load(static_cast(nullptr))); + constexpr auto quoteMask = SIMD::splat('"'); + constexpr auto escapeMask = SIMD::splat('\\'); + constexpr auto controlMask = SIMD::splat(' '); + constexpr auto zeros = SIMD::splat(0); + const auto* ptr = span.data(); + const auto* end = ptr + span.size(); + auto* cursorEnd = cursor + span.size(); + BulkType accumulated { }; + for (; ptr + (stride - 1) < end; ptr += stride, cursor += stride) { + auto input = SIMD::load(bitwise_cast(ptr)); + simde_vst2q_u8(bitwise_cast(cursor), (simde_uint8x16x2_t { input, zeros })); + auto quotes = SIMD::equal(input, quoteMask); + auto escapes = SIMD::equal(input, escapeMask); + auto controls = SIMD::lessThan(input, controlMask); + accumulated = SIMD::bitOr(accumulated, quotes, escapes, controls); + } + if (ptr < end) { + auto input = SIMD::load(bitwise_cast(end - stride)); + simde_vst2q_u8(bitwise_cast(cursorEnd - stride), (simde_uint8x16x2_t { input, zeros })); + auto quotes = SIMD::equal(input, quoteMask); + auto escapes = SIMD::equal(input, escapeMask); + auto controls = SIMD::lessThan(input, controlMask); + accumulated = SIMD::bitOr(accumulated, quotes, escapes, controls); + } + return SIMD::isNonZero(accumulated); + } +#endif + for (auto character : span) { + if (UNLIKELY(WTF::escapedFormsForJSON[character])) + return true; + *cursor++ = character; + } + return false; +} + template void FastStringifier::append(JSValue value) { @@ -1130,101 +1228,6 @@ void FastStringifier::append(JSValue value) return; } - auto charactersCopySameType = [&](auto span, auto* cursor) ALWAYS_INLINE_LAMBDA { -#if (CPU(ARM64) || CPU(X86_64)) && COMPILER(CLANG) - constexpr size_t stride = SIMD::stride; - if (span.size() >= stride) { - using UnsignedType = std::make_unsigned_t; - using BulkType = decltype(SIMD::load(static_cast(nullptr))); - constexpr auto quoteMask = SIMD::splat('"'); - constexpr auto escapeMask = SIMD::splat('\\'); - constexpr auto controlMask = SIMD::splat(' '); - const auto* ptr = span.data(); - const auto* end = ptr + span.size(); - auto* cursorEnd = cursor + span.size(); - BulkType accumulated { }; - for (; ptr + (stride - 1) < end; ptr += stride, cursor += stride) { - auto input = SIMD::load(bitwise_cast(ptr)); - SIMD::store(input, bitwise_cast(cursor)); - auto quotes = SIMD::equal(input, quoteMask); - auto escapes = SIMD::equal(input, escapeMask); - auto controls = SIMD::lessThan(input, controlMask); - accumulated = SIMD::bitOr(accumulated, quotes, escapes, controls); - if constexpr (sizeof(CharType) != 1) { - constexpr auto surrogateMask = SIMD::splat(0xf800); - constexpr auto surrogateCheckMask = SIMD::splat(0xd800); - accumulated = SIMD::bitOr(accumulated, SIMD::equal(SIMD::bitAnd(input, surrogateMask), surrogateCheckMask)); - } - } - if (ptr < end) { - auto input = SIMD::load(bitwise_cast(end - stride)); - SIMD::store(input, bitwise_cast(cursorEnd - stride)); - auto quotes = SIMD::equal(input, quoteMask); - auto escapes = SIMD::equal(input, escapeMask); - auto controls = SIMD::lessThan(input, controlMask); - accumulated = SIMD::bitOr(accumulated, quotes, escapes, controls); - if constexpr (sizeof(CharType) != 1) { - constexpr auto surrogateMask = SIMD::splat(0xf800); - constexpr auto surrogateCheckMask = SIMD::splat(0xd800); - accumulated = SIMD::bitOr(accumulated, SIMD::equal(SIMD::bitAnd(input, surrogateMask), surrogateCheckMask)); - } - } - return SIMD::isNonZero(accumulated); - } -#endif - for (auto character : span) { - if constexpr (sizeof(CharType) != 1) { - if (UNLIKELY(U16_IS_SURROGATE(character))) - return true; - } - if (UNLIKELY(character <= 0xff && WTF::escapedFormsForJSON[character])) - return true; - *cursor++ = character; - } - return false; - }; - - auto charactersCopyUpconvert = [&](std::span span, UChar* cursor) ALWAYS_INLINE_LAMBDA { -#if (CPU(ARM64) || CPU(X86_64)) && COMPILER(CLANG) - constexpr size_t stride = SIMD::stride; - if (span.size() >= stride) { - using UnsignedType = std::make_unsigned_t; - using BulkType = decltype(SIMD::load(static_cast(nullptr))); - constexpr auto quoteMask = SIMD::splat('"'); - constexpr auto escapeMask = SIMD::splat('\\'); - constexpr auto controlMask = SIMD::splat(' '); - constexpr auto zeros = SIMD::splat(0); - const auto* ptr = span.data(); - const auto* end = ptr + span.size(); - auto* cursorEnd = cursor + span.size(); - BulkType accumulated { }; - for (; ptr + (stride - 1) < end; ptr += stride, cursor += stride) { - auto input = SIMD::load(bitwise_cast(ptr)); - simde_vst2q_u8(bitwise_cast(cursor), (simde_uint8x16x2_t { input, zeros })); - auto quotes = SIMD::equal(input, quoteMask); - auto escapes = SIMD::equal(input, escapeMask); - auto controls = SIMD::lessThan(input, controlMask); - accumulated = SIMD::bitOr(accumulated, quotes, escapes, controls); - } - if (ptr < end) { - auto input = SIMD::load(bitwise_cast(end - stride)); - simde_vst2q_u8(bitwise_cast(cursorEnd - stride), (simde_uint8x16x2_t { input, zeros })); - auto quotes = SIMD::equal(input, quoteMask); - auto escapes = SIMD::equal(input, escapeMask); - auto controls = SIMD::lessThan(input, controlMask); - accumulated = SIMD::bitOr(accumulated, quotes, escapes, controls); - } - return SIMD::isNonZero(accumulated); - } -#endif - for (auto character : span) { - if (UNLIKELY(WTF::escapedFormsForJSON[character])) - return true; - *cursor++ = character; - } - return false; - }; - auto stringLength = string.data.length(); if constexpr (sizeof(CharType) == 1) { if (UNLIKELY(!string.data.is8Bit())) { @@ -1239,7 +1242,7 @@ void FastStringifier::append(JSValue value) return; } buffer()[m_length] = '"'; - if (LIKELY(!charactersCopySameType(string.data.span8(), buffer() + m_length + 1))) { + if (LIKELY(!stringCopySameType(string.data.span8(), buffer() + m_length + 1))) { buffer()[m_length + 1 + stringLength] = '"'; m_length += 1 + stringLength + 1; return; @@ -1251,13 +1254,13 @@ void FastStringifier::append(JSValue value) } buffer()[m_length] = '"'; if (string.data.is8Bit()) { - if (LIKELY(!charactersCopyUpconvert(string.data.span8(), buffer() + m_length + 1))) { + if (LIKELY(!stringCopyUpconvert(string.data.span8(), buffer() + m_length + 1))) { buffer()[m_length + 1 + stringLength] = '"'; m_length += 1 + stringLength + 1; return; } } else { - if (LIKELY(!charactersCopySameType(string.data.span16(), buffer() + m_length + 1))) { + if (LIKELY(!stringCopySameType(string.data.span16(), buffer() + m_length + 1))) { buffer()[m_length + 1 + stringLength] = '"'; m_length += 1 + stringLength + 1; return; @@ -1328,6 +1331,7 @@ void FastStringifier::append(JSValue value) recordFailure("16-bit property name"_s); return false; } + auto span = name.span8(); if (UNLIKELY(object.structure() != &structure)) { ASSERT_NOT_REACHED(); @@ -1339,26 +1343,29 @@ void FastStringifier::append(JSValue value) return true; bool needComma = buffer()[m_length - 1] != '{'; - unsigned nameLength = name.length(); - if (UNLIKELY(!hasRemainingCapacity(needComma + 1 + nameLength + 2))) { + if (UNLIKELY(!hasRemainingCapacity(needComma + 1 + span.size() + 2))) { recordBufferFull(); return false; } if (needComma) buffer()[m_length++] = ','; buffer()[m_length] = '"'; - auto characters = name.span8(); - for (unsigned i = 0; i < nameLength; ++i) { - auto character = characters[i]; - if (UNLIKELY(WTF::escapedFormsForJSON[character])) { + + if constexpr (sizeof(CharType) == 2) { + if (UNLIKELY(stringCopyUpconvert(span, buffer() + m_length + 1))) { + recordFailure("property name character needs escaping"_s); + return false; + } + } else { + if (UNLIKELY(stringCopySameType(span, buffer() + m_length + 1))) { recordFailure("property name character needs escaping"_s); return false; } - buffer()[m_length + 1 + i] = character; } - buffer()[m_length + 1 + nameLength] = '"'; - buffer()[m_length + 1 + nameLength + 1] = ':'; - m_length += 1 + nameLength + 2; + + buffer()[m_length + 1 + span.size()] = '"'; + buffer()[m_length + 1 + span.size() + 1] = ':'; + m_length += 1 + span.size() + 2; append(value); return !haveFailure(); });