From f0e8af72aed61a6163aa88d741c98480f27d4e5c Mon Sep 17 00:00:00 2001 From: santorac <55155825+santorac@users.noreply.github.com> Date: Thu, 30 Sep 2021 01:08:54 -0700 Subject: [PATCH] Updated StringFunc::Tokenize to support returning a list of string_view instead of string, which should be more efficient. string is still supported as well, but users should prefer the string_view version. Testing: Updated unit tests. Reprocessed Atom material assets. Ran AtomSampleViewer material screenshot test. Opened, edited, saved materail in the Material Editor. Opened a level, edited material property overrides, saved and reloaded. Signed-off-by: santorac <55155825+santorac@users.noreply.github.com> --- .../AzCore/AzCore/StringFunc/StringFunc.cpp | 24 ++++++++++---- .../AzCore/AzCore/StringFunc/StringFunc.h | 12 ++++--- Code/Framework/AzCore/Tests/StringFunc.cpp | 31 +++++++++++++++---- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.cpp b/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.cpp index ff30291a70..eec23b6bcb 100644 --- a/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.cpp +++ b/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.cpp @@ -759,13 +759,18 @@ namespace AZ } return value; } - - void Tokenize(AZStd::string_view in, AZStd::vector& tokens, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings) + + template + void Tokenize(AZStd::string_view in, AZStd::vector& tokens, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings) { return Tokenize(in, tokens, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings); } - void Tokenize(AZStd::string_view in, AZStd::vector& tokens, AZStd::string_view delimiters, bool keepEmptyStrings, bool keepSpaceStrings) + template void Tokenize(AZStd::string_view in, AZStd::vector& tokens, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings); + template void Tokenize(AZStd::string_view in, AZStd::vector& tokens, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings); + + template + void Tokenize(AZStd::string_view in, AZStd::vector& tokens, AZStd::string_view delimiters, bool keepEmptyStrings, bool keepSpaceStrings) { auto insertVisitor = [&tokens](AZStd::string_view token) { @@ -773,6 +778,9 @@ namespace AZ }; return TokenizeVisitor(in, insertVisitor, delimiters, keepEmptyStrings, keepSpaceStrings); } + + template void Tokenize(AZStd::string_view in, AZStd::vector& tokens, AZStd::string_view delimiters, bool keepEmptyStrings, bool keepSpaceStrings); + template void Tokenize(AZStd::string_view in, AZStd::vector& tokens, AZStd::string_view delimiters, bool keepEmptyStrings, bool keepSpaceStrings); void TokenizeVisitor(AZStd::string_view in, const TokenVisitor& tokenVisitor, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings) { @@ -920,8 +928,9 @@ namespace AZ return found; } - - void Tokenize(AZStd::string_view input, AZStd::vector& tokens, const AZStd::vector& delimiters, bool keepEmptyStrings /*= false*/, bool keepSpaceStrings /*= false*/) + + template + void Tokenize(AZStd::string_view input, AZStd::vector& tokens, const AZStd::vector& delimiters, bool keepEmptyStrings /*= false*/, bool keepSpaceStrings /*= false*/) { if (input.empty()) { @@ -941,7 +950,7 @@ namespace AZ } // Take the substring, not including the separator, and increment our offset - AZStd::string nextSubstring = input.substr(offset, nextOffset - offset); + AZStd::string_view nextSubstring = input.substr(offset, nextOffset - offset); if (keepEmptyStrings || keepSpaceStrings || !nextSubstring.empty()) { tokens.push_back(nextSubstring); @@ -950,6 +959,9 @@ namespace AZ offset = nextOffset + delimiters[nextMatch].size(); } } + + template void Tokenize(AZStd::string_view input, AZStd::vector& tokens, const AZStd::vector& delimiters, bool keepEmptyStrings /*= false*/, bool keepSpaceStrings /*= false*/); + template void Tokenize(AZStd::string_view input, AZStd::vector& tokens, const AZStd::vector& delimiters, bool keepEmptyStrings /*= false*/, bool keepSpaceStrings /*= false*/); int ToInt(const char* in) { diff --git a/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.h b/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.h index 1e651afc93..51357312bc 100644 --- a/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.h +++ b/Code/Framework/AzCore/AzCore/StringFunc/StringFunc.h @@ -258,17 +258,21 @@ namespace AZ bool Strip(AZStd::string& inout, const char* stripCharacters = " ", bool bCaseSensitive = false, bool bStripBeginning = false, bool bStripEnding = false); //! Tokenize - /*! Tokenize a c-string, into a vector of AZStd::string(s) optionally keeping empty string + /*! Tokenize a c-string, into a vector of strings optionally keeping empty string *! and optionally keeping space only strings + *! (The string type may be AZStd::string or AZStd::string_view. New code should use AZStd::string_view for better performance. AZStd::string version is preserved for compatibility.) Example: Tokenize the words of a sentence. StringFunc::Tokenize("Hello World", d, ' '); s[0] == "Hello", s[1] == "World" Example: Tokenize a comma and end line delimited string StringFunc::Tokenize("Hello,World\nHello,World", d, ' '); s[0] == "Hello", s[1] == "World" s[2] == "Hello", s[3] == "World" */ - void Tokenize(AZStd::string_view in, AZStd::vector& tokens, const char delimiter, bool keepEmptyStrings = false, bool keepSpaceStrings = false); - void Tokenize(AZStd::string_view in, AZStd::vector& tokens, AZStd::string_view delimiters = "\\//, \t\n", bool keepEmptyStrings = false, bool keepSpaceStrings = false); - void Tokenize(AZStd::string_view in, AZStd::vector& tokens, const AZStd::vector& delimiters, bool keepEmptyStrings = false, bool keepSpaceStrings = false); + template + void Tokenize(AZStd::string_view in, AZStd::vector& tokens, const char delimiter, bool keepEmptyStrings = false, bool keepSpaceStrings = false); + template + void Tokenize(AZStd::string_view in, AZStd::vector& tokens, AZStd::string_view delimiters = "\\//, \t\n", bool keepEmptyStrings = false, bool keepSpaceStrings = false); + template + void Tokenize(AZStd::string_view in, AZStd::vector& tokens, const AZStd::vector& delimiters, bool keepEmptyStrings = false, bool keepSpaceStrings = false); //! TokenizeVisitor /*! Tokenize a string_view and invoke a handler for each token found. diff --git a/Code/Framework/AzCore/Tests/StringFunc.cpp b/Code/Framework/AzCore/Tests/StringFunc.cpp index 68821ba3f9..79dabe3462 100644 --- a/Code/Framework/AzCore/Tests/StringFunc.cpp +++ b/Code/Framework/AzCore/Tests/StringFunc.cpp @@ -199,7 +199,7 @@ namespace AZ TEST_F(StringFuncTest, Tokenize_SingleDelimeter_Empty) { AZStd::string input = ""; - AZStd::vector tokens; + AZStd::vector tokens; AZ::StringFunc::Tokenize(input.c_str(), tokens, ' '); ASSERT_EQ(tokens.size(), 0); } @@ -207,7 +207,7 @@ namespace AZ TEST_F(StringFuncTest, Tokenize_SingleDelimeter) { AZStd::string input = "a b,c"; - AZStd::vector tokens; + AZStd::vector tokens; AZ::StringFunc::Tokenize(input.c_str(), tokens, ' '); ASSERT_EQ(tokens.size(), 2); @@ -218,7 +218,7 @@ namespace AZ TEST_F(StringFuncTest, Tokenize_MultiDelimeter_Empty) { AZStd::string input = ""; - AZStd::vector tokens; + AZStd::vector tokens; AZ::StringFunc::Tokenize(input.c_str(), tokens, " ,"); ASSERT_EQ(tokens.size(), 0); } @@ -226,7 +226,7 @@ namespace AZ TEST_F(StringFuncTest, Tokenize_MultiDelimeters) { AZStd::string input = " -a +b +c -d-e"; - AZStd::vector tokens; + AZStd::vector tokens; AZ::StringFunc::Tokenize(input.c_str(), tokens, "-+"); ASSERT_EQ(tokens.size(), 5); @@ -240,7 +240,7 @@ namespace AZ TEST_F(StringFuncTest, Tokenize_SubstringDelimeters_Empty) { AZStd::string input = ""; - AZStd::vector tokens; + AZStd::vector tokens; AZStd::vector delimeters = {" -", " +"}; AZ::StringFunc::Tokenize(input.c_str(), tokens, delimeters); ASSERT_EQ(tokens.size(), 0); @@ -249,7 +249,7 @@ namespace AZ TEST_F(StringFuncTest, Tokenize_SubstringDelimeters) { AZStd::string input = " -a +b +c -d-e"; - AZStd::vector tokens; + AZStd::vector tokens; AZStd::vector delimeters = { " -", " +" }; AZ::StringFunc::Tokenize(input.c_str(), tokens, delimeters); @@ -259,6 +259,25 @@ namespace AZ ASSERT_TRUE(tokens[2] == "c"); ASSERT_TRUE(tokens[3] == "d-e"); // Test for something like a guid, which contain typical separator characters } + + TEST_F(StringFuncTest, Tokenize_MultiDelimeters_String) + { + // Test with AZStd::string for backward compatibility. The functions + // use to only work with AZStd::string, and now they are templatized + // to support both AZStd::string and AZStd::string_view (the latter + // being perferred for performance). + + AZStd::string input = " -a +b +c -d-e"; + AZStd::vector tokens; + AZ::StringFunc::Tokenize(input.c_str(), tokens, "-+"); + + ASSERT_EQ(tokens.size(), 5); + ASSERT_TRUE(tokens[0] == "a "); + ASSERT_TRUE(tokens[1] == "b "); + ASSERT_TRUE(tokens[2] == "c "); + ASSERT_TRUE(tokens[3] == "d"); + ASSERT_TRUE(tokens[4] == "e"); + } TEST_F(StringFuncTest, TokenizeVisitor_EmptyString_DoesNotInvokeVisitor) {