birkeh
/
o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reverted the new Tokenize function I added, and used TokenizeVisitor instead.

Browse Source 
Signed-off-by: santorac <55155825+santorac@users.noreply.github.com>
monroegm-disable-blank-issue-2
santorac 4 years ago
parent 6caeabd961
commit 68e4970a2d
4 changed files with 24 additions and 53 deletions
Show Stats Download Patch File Download Diff File

20
Code/Framework/AzCore/AzCore/StringFunc/StringFunc.cpp
Unescape Escape View File

@ -758,17 +758,12 @@ namespace AZ::StringFunc
return value; return value;
} }
template<typename StringType> void Tokenize(AZStd::string_view in, AZStd::vector<AZStd::string>& tokens, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings)
void Tokenize(AZStd::string_view in, AZStd::vector<StringType>& tokens, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings)
{ {
return Tokenize(in, tokens, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings); return Tokenize(in, tokens, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings);
} }
template void Tokenize(AZStd::string_view in, AZStd::vector<AZStd::string>& tokens, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings); void Tokenize(AZStd::string_view in, AZStd::vector<AZStd::string>& tokens, AZStd::string_view delimiters, bool keepEmptyStrings, bool keepSpaceStrings)
template void Tokenize(AZStd::string_view in, AZStd::vector<AZStd::string_view>& tokens, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings);
template<typename StringType>
void Tokenize(AZStd::string_view in, AZStd::vector<StringType>& tokens, AZStd::string_view delimiters, bool keepEmptyStrings, bool keepSpaceStrings)
{ {
auto insertVisitor = [&tokens](AZStd::string_view token) auto insertVisitor = [&tokens](AZStd::string_view token)
{ {
@ -777,9 +772,6 @@ namespace AZ::StringFunc
return TokenizeVisitor(in, insertVisitor, delimiters, keepEmptyStrings, keepSpaceStrings); return TokenizeVisitor(in, insertVisitor, delimiters, keepEmptyStrings, keepSpaceStrings);
} }
template void Tokenize(AZStd::string_view in, AZStd::vector<AZStd::string>& tokens, AZStd::string_view delimiters, bool keepEmptyStrings, bool keepSpaceStrings);
template void Tokenize(AZStd::string_view in, AZStd::vector<AZStd::string_view>& tokens, AZStd::string_view delimiters, bool keepEmptyStrings, bool keepSpaceStrings);
void TokenizeVisitor(AZStd::string_view in, const TokenVisitor& tokenVisitor, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings) void TokenizeVisitor(AZStd::string_view in, const TokenVisitor& tokenVisitor, const char delimiter, bool keepEmptyStrings, bool keepSpaceStrings)
{ {
return TokenizeVisitor(in, tokenVisitor, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings); return TokenizeVisitor(in, tokenVisitor, { &delimiter, 1 }, keepEmptyStrings, keepSpaceStrings);
@ -927,8 +919,7 @@ namespace AZ::StringFunc
return found; return found;
} }
template<typename StringType> void Tokenize(AZStd::string_view input, AZStd::vector<AZStd::string>& tokens, const AZStd::vector<AZStd::string_view>& delimiters, bool keepEmptyStrings /*= false*/, bool keepSpaceStrings /*= false*/)
void Tokenize(AZStd::string_view input, AZStd::vector<StringType>& tokens, const AZStd::vector<AZStd::string_view>& delimiters, bool keepEmptyStrings /*= false*/, bool keepSpaceStrings /*= false*/)
{ {
if (input.empty()) if (input.empty())
{ {
@ -948,7 +939,7 @@ namespace AZ::StringFunc
} }
// Take the substring, not including the separator, and increment our offset // Take the substring, not including the separator, and increment our offset
AZStd::string_view nextSubstring = input.substr(offset, nextOffset - offset); AZStd::string nextSubstring = input.substr(offset, nextOffset - offset);
if (keepEmptyStrings || keepSpaceStrings || !nextSubstring.empty()) if (keepEmptyStrings || keepSpaceStrings || !nextSubstring.empty())
{ {
tokens.push_back(nextSubstring); tokens.push_back(nextSubstring);
@ -958,9 +949,6 @@ namespace AZ::StringFunc
} }
} }
template void Tokenize(AZStd::string_view input, AZStd::vector<AZStd::string>& tokens, const AZStd::vector<AZStd::string_view>& delimiters, bool keepEmptyStrings /*= false*/, bool keepSpaceStrings /*= false*/);
template void Tokenize(AZStd::string_view input, AZStd::vector<AZStd::string_view>& tokens, const AZStd::vector<AZStd::string_view>& delimiters, bool keepEmptyStrings /*= false*/, bool keepSpaceStrings /*= false*/);
int ToInt(const char* in) int ToInt(const char* in)
{ {
if (!in) if (!in)

12
Code/Framework/AzCore/AzCore/StringFunc/StringFunc.h
Unescape Escape View File

@ -258,21 +258,17 @@ namespace AZ
bool Strip(AZStd::string& inout, const char* stripCharacters = " ", bool bCaseSensitive = false, bool bStripBeginning = false, bool bStripEnding = false); bool Strip(AZStd::string& inout, const char* stripCharacters = " ", bool bCaseSensitive = false, bool bStripBeginning = false, bool bStripEnding = false);
//! Tokenize //! Tokenize
/*! Tokenize a c-string, into a vector of strings optionally keeping empty string /*! Tokenize a c-string, into a vector of AZStd::string(s) optionally keeping empty string
*! and optionally keeping space only strings *! and optionally keeping space only strings
*! (The string type may be AZStd::string or AZStd::string_view. New code should use AZStd::string_view for better performance. AZStd::string version is preserved for compatibility.)
Example: Tokenize the words of a sentence. Example: Tokenize the words of a sentence.
StringFunc::Tokenize("Hello World", d, ' '); s[0] == "Hello", s[1] == "World" StringFunc::Tokenize("Hello World", d, ' '); s[0] == "Hello", s[1] == "World"
Example: Tokenize a comma and end line delimited string Example: Tokenize a comma and end line delimited string
StringFunc::Tokenize("Hello,World\nHello,World", d, ' '); s[0] == "Hello", s[1] == "World" StringFunc::Tokenize("Hello,World\nHello,World", d, ' '); s[0] == "Hello", s[1] == "World"
s[2] == "Hello", s[3] == "World" s[2] == "Hello", s[3] == "World"
*/ */
template<typename StringType> void Tokenize(AZStd::string_view in, AZStd::vector<AZStd::string>& tokens, const char delimiter, bool keepEmptyStrings = false, bool keepSpaceStrings = false);
void Tokenize(AZStd::string_view in, AZStd::vector<StringType>& tokens, const char delimiter, bool keepEmptyStrings = false, bool keepSpaceStrings = false); void Tokenize(AZStd::string_view in, AZStd::vector<AZStd::string>& tokens, AZStd::string_view delimiters = "\\//, \t\n", bool keepEmptyStrings = false, bool keepSpaceStrings = false);
template<typename StringType> void Tokenize(AZStd::string_view in, AZStd::vector<AZStd::string>& tokens, const AZStd::vector<AZStd::string_view>& delimiters, bool keepEmptyStrings = false, bool keepSpaceStrings = false);
void Tokenize(AZStd::string_view in, AZStd::vector<StringType>& tokens, AZStd::string_view delimiters = "\\//, \t\n", bool keepEmptyStrings = false, bool keepSpaceStrings = false);
template<typename StringType>
void Tokenize(AZStd::string_view in, AZStd::vector<StringType>& tokens, const AZStd::vector<AZStd::string_view>& delimiters, bool keepEmptyStrings = false, bool keepSpaceStrings = false);
//! TokenizeVisitor //! TokenizeVisitor
/*! Tokenize a string_view and invoke a handler for each token found. /*! Tokenize a string_view and invoke a handler for each token found.

31
Code/Framework/AzCore/Tests/StringFunc.cpp
Unescape Escape View File

@ -199,7 +199,7 @@ namespace AZ
TEST_F(StringFuncTest, Tokenize_SingleDelimeter_Empty) TEST_F(StringFuncTest, Tokenize_SingleDelimeter_Empty)
{ {
AZStd::string input = ""; AZStd::string input = "";
AZStd::vector<AZStd::string_view> tokens; AZStd::vector<AZStd::string> tokens;
AZ::StringFunc::Tokenize(input.c_str(), tokens, ' '); AZ::StringFunc::Tokenize(input.c_str(), tokens, ' ');
ASSERT_EQ(tokens.size(), 0); ASSERT_EQ(tokens.size(), 0);
} }
@ -207,7 +207,7 @@ namespace AZ
TEST_F(StringFuncTest, Tokenize_SingleDelimeter) TEST_F(StringFuncTest, Tokenize_SingleDelimeter)
{ {
AZStd::string input = "a b,c"; AZStd::string input = "a b,c";
AZStd::vector<AZStd::string_view> tokens; AZStd::vector<AZStd::string> tokens;
AZ::StringFunc::Tokenize(input.c_str(), tokens, ' '); AZ::StringFunc::Tokenize(input.c_str(), tokens, ' ');
ASSERT_EQ(tokens.size(), 2); ASSERT_EQ(tokens.size(), 2);
@ -218,7 +218,7 @@ namespace AZ
TEST_F(StringFuncTest, Tokenize_MultiDelimeter_Empty) TEST_F(StringFuncTest, Tokenize_MultiDelimeter_Empty)
{ {
AZStd::string input = ""; AZStd::string input = "";
AZStd::vector<AZStd::string_view> tokens; AZStd::vector<AZStd::string> tokens;
AZ::StringFunc::Tokenize(input.c_str(), tokens, " ,"); AZ::StringFunc::Tokenize(input.c_str(), tokens, " ,");
ASSERT_EQ(tokens.size(), 0); ASSERT_EQ(tokens.size(), 0);
} }
@ -226,7 +226,7 @@ namespace AZ
TEST_F(StringFuncTest, Tokenize_MultiDelimeters) TEST_F(StringFuncTest, Tokenize_MultiDelimeters)
{ {
AZStd::string input = " -a +b +c -d-e"; AZStd::string input = " -a +b +c -d-e";
AZStd::vector<AZStd::string_view> tokens; AZStd::vector<AZStd::string> tokens;
AZ::StringFunc::Tokenize(input.c_str(), tokens, "-+"); AZ::StringFunc::Tokenize(input.c_str(), tokens, "-+");
ASSERT_EQ(tokens.size(), 5); ASSERT_EQ(tokens.size(), 5);
@ -240,7 +240,7 @@ namespace AZ
TEST_F(StringFuncTest, Tokenize_SubstringDelimeters_Empty) TEST_F(StringFuncTest, Tokenize_SubstringDelimeters_Empty)
{ {
AZStd::string input = ""; AZStd::string input = "";
AZStd::vector<AZStd::string_view> tokens; AZStd::vector<AZStd::string> tokens;
AZStd::vector<AZStd::string_view> delimeters = {" -", " +"}; AZStd::vector<AZStd::string_view> delimeters = {" -", " +"};
AZ::StringFunc::Tokenize(input.c_str(), tokens, delimeters); AZ::StringFunc::Tokenize(input.c_str(), tokens, delimeters);
ASSERT_EQ(tokens.size(), 0); ASSERT_EQ(tokens.size(), 0);
@ -249,7 +249,7 @@ namespace AZ
TEST_F(StringFuncTest, Tokenize_SubstringDelimeters) TEST_F(StringFuncTest, Tokenize_SubstringDelimeters)
{ {
AZStd::string input = " -a +b +c -d-e"; AZStd::string input = " -a +b +c -d-e";
AZStd::vector<AZStd::string_view> tokens; AZStd::vector<AZStd::string> tokens;
AZStd::vector<AZStd::string_view> delimeters = { " -", " +" }; AZStd::vector<AZStd::string_view> delimeters = { " -", " +" };
AZ::StringFunc::Tokenize(input.c_str(), tokens, delimeters); AZ::StringFunc::Tokenize(input.c_str(), tokens, delimeters);
@ -260,25 +260,6 @@ namespace AZ
ASSERT_TRUE(tokens[3] == "d-e"); // Test for something like a guid, which contain typical separator characters ASSERT_TRUE(tokens[3] == "d-e"); // Test for something like a guid, which contain typical separator characters
} }
TEST_F(StringFuncTest, Tokenize_MultiDelimeters_String)
{
// Test with AZStd::string for backward compatibility. The functions
// use to only work with AZStd::string, and now they are templatized
// to support both AZStd::string and AZStd::string_view (the latter
// being perferred for performance).
AZStd::string input = " -a +b +c -d-e";
AZStd::vector<AZStd::string> tokens;
AZ::StringFunc::Tokenize(input.c_str(), tokens, "-+");
ASSERT_EQ(tokens.size(), 5);
ASSERT_TRUE(tokens[0] == "a ");
ASSERT_TRUE(tokens[1] == "b ");
ASSERT_TRUE(tokens[2] == "c ");
ASSERT_TRUE(tokens[3] == "d");
ASSERT_TRUE(tokens[4] == "e");
}
TEST_F(StringFuncTest, TokenizeVisitor_EmptyString_DoesNotInvokeVisitor) TEST_F(StringFuncTest, TokenizeVisitor_EmptyString_DoesNotInvokeVisitor)
{ {
int visitedCount{}; int visitedCount{};

8
Gems/Atom/RPI/Code/Source/RPI.Edit/Material/MaterialTypeSourceData.cpp
Unescape Escape View File

@ -327,7 +327,13 @@ namespace AZ
AZStd::vector<AZStd::string_view> MaterialTypeSourceData::TokenizeId(AZStd::string_view id) AZStd::vector<AZStd::string_view> MaterialTypeSourceData::TokenizeId(AZStd::string_view id)
{ {
AZStd::vector<AZStd::string_view> tokens; AZStd::vector<AZStd::string_view> tokens;
AzFramework::StringFunc::Tokenize(id, tokens, "./", true, true);
AzFramework::StringFunc::TokenizeVisitor(id, [&tokens](AZStd::string_view t)
{
tokens.push_back(t);
},
"./", true, true);
return tokens; return tokens;
} }

Write Preview
Loading…
Cancel
Save