birkeh
/
o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Restore fix lost in merge: [LY-121929] Asset Processor Error : Sqlite - Failed to prepare statement. (#2202)

Browse Source 
Fixed unresolved product dependency query to be able to handle much larger queries (tested with 100,000 products) by storing queried products in a temp table first

Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>
monroegm-disable-blank-issue-2
amzn-mike 5 years ago committed by GitHub
parent ccd4cc65a5
commit 8116e239f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 145 additions and 76 deletions
Show Stats Download Patch File Download Diff File

155
Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.cpp
Unescape Escape View File

@ -850,6 +850,32 @@ namespace AzToolsFramework
"SELECT * FROM ProductDependencies WHERE UnresolvedPath LIKE \":%\"";
static const auto s_queryProductDependencyExclusions = MakeSqlQuery(GET_PRODUCT_DEPENDENCY_EXCLUSIONS, GET_PRODUCT_DEPENDENCY_EXCLUSIONS_STATEMENT, LOG_NAME);
static const char* CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE =
"AssetProcessor::CreateUnresolvedProductDependenciesTempTable";
static const char* CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE_STATEMENT =
"CREATE TEMPORARY TABLE QueryProductDependenciesUnresolvedAdvanced(search)";
static const auto s_createUnresolvedProductDependenciesTempTable = MakeSqlQuery(
CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE, CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE_STATEMENT, LOG_NAME);
static const char* INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES = "AssetProcessor::InsertProductDependencyTempTableValues";
static const char* INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES_STATEMENT =
"INSERT INTO QueryProductDependenciesUnresolvedAdvanced VALUES (:filename)";
static const auto s_queryInsertProductDependencyTempTableValues = MakeSqlQuery(
INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES,
INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES_STATEMENT,
LOG_NAME,
SqlParam<const char*>(":filename"));
static const char* GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE =
"AssetProcessor::GetUnresolvedProductDependenciesUsingTempTable";
static const char* GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE_STATEMENT =
"SELECT * FROM ProductDependencies INNER JOIN QueryProductDependenciesUnresolvedAdvanced "
"ON (UnresolvedPath LIKE \"%*%\" AND search LIKE REPLACE(UnresolvedPath, \"*\", \"%\")) OR search = UnresolvedPath";
static const auto s_queryGetUnresolvedProductDependenciesUsingTempTable = MakeSqlQuery(
GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE,
GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE_STATEMENT,
LOG_NAME);
// lookup by primary key
static const char* QUERY_FILE_BY_FILEID = "AzToolsFramework::AssetDatabase::QueryFileByFileID";
static const char* QUERY_FILE_BY_FILEID_STATEMENT =
@ -1814,6 +1840,9 @@ namespace AzToolsFramework
AddStatement(m_databaseConnection, s_queryAllProductdependencies);
AddStatement(m_databaseConnection, s_queryUnresolvedProductDependencies);
AddStatement(m_databaseConnection, s_queryProductDependencyExclusions);
AddStatement(m_databaseConnection, s_createUnresolvedProductDependenciesTempTable);
AddStatement(m_databaseConnection, s_queryInsertProductDependencyTempTableValues);
AddStatement(m_databaseConnection, s_queryGetUnresolvedProductDependenciesUsingTempTable);
AddStatement(m_databaseConnection, s_queryFileByFileid);
AddStatement(m_databaseConnection, s_queryFilesByFileName);
@ -2480,88 +2509,26 @@ namespace AzToolsFramework
return s_queryProductDependencyExclusions.BindAndQuery(*m_databaseConnection, handler, &GetProductDependencyResult);
}
bool AssetDatabaseConnection::QueryProductDependenciesUnresolvedAdvanced(const AZStd::vector<AZStd::string>& searchPaths, productDependencyAndPathHandler handler)
bool AssetDatabaseConnection::QueryProductDependenciesUnresolvedAdvanced(
const AZStd::vector<AZStd::string>& searchPaths, productDependencyAndPathHandler handler)
{
AZStd::string sql = R"END(select c.*, b.search FROM (select REPLACE(UnresolvedPath, "*", "%") as search, ProductDependencyID from ProductDependencies where UnresolvedPath LIKE "%*%") as a, ()END";
ScopedTransaction transaction(m_databaseConnection);
bool first = true;
bool result = s_createUnresolvedProductDependenciesTempTable.BindAndStep(*m_databaseConnection);
for(int i = 0; i < searchPaths.size(); ++i)
for (auto&& path : searchPaths)
{
if(first)
{
sql += "SELECT ? as search ";
first = false;
}
else
{
sql += "UNION SELECT ? ";
}
}
sql += R"END() as b
INNER JOIN ProductDependencies as c ON c.ProductDependencyID = a.ProductDependencyID
WHERE b.search LIKE a.search
UNION SELECT p.*, UnresolvedPath
FROM ProductDependencies as p
WHERE UnresolvedPath != ""
)END";
first = true;
for (int i = 0; i < searchPaths.size(); ++i)
{
if(first)
{
sql += " AND ";
first = false;
}
else
{
sql += " OR ";
}
sql += "UnresolvedPath = ?";
result = s_queryInsertProductDependencyTempTableValues.BindAndStep(*m_databaseConnection, path.c_str()) && result;
}
bool result = m_databaseConnection->ExecuteRawSqlQuery(sql, [handler](sqlite3_stmt* statement)
{
ProductDependencyDatabaseEntry entry;
entry.m_productDependencyID = SQLite::GetColumnInt64(statement, 0);
entry.m_productPK = SQLite::GetColumnInt64(statement, 1);
entry.m_dependencySourceGuid = SQLite::GetColumnUuid(statement, 2);
entry.m_dependencySubID = GetColumnInt(statement, 3);
entry.m_platform = GetColumnText(statement, 4);
entry.m_dependencyFlags = GetColumnInt64(statement, 5);
entry.m_unresolvedPath = GetColumnText(statement, 6);
entry.m_dependencyType = static_cast<ProductDependencyDatabaseEntry::DependencyType>(GetColumnInt(statement, 7));
entry.m_fromAssetId = sqlite3_column_int(statement, 8);
AZStd::string matchedPath = GetColumnText(statement, 9);
result = s_queryGetUnresolvedProductDependenciesUsingTempTable.BindAndQuery(
*m_databaseConnection, handler, &GetProductDependencyAndPathResult) &&
result;
handler(entry, matchedPath);
result = m_databaseConnection->ExecuteRawSqlQuery("DROP TABLE QueryProductDependenciesUnresolvedAdvanced", nullptr, nullptr) &&
result;
return true;
}, [&searchPaths](sqlite3_stmt* statement)
{
int index = 1;
for (const auto& path : searchPaths)
{
[[maybe_unused]] int res = sqlite3_bind_text(statement, index, path.c_str(), static_cast<int>(path.size()), nullptr);
AZ_Assert(res == SQLITE_OK, "Statement::BindValueText: failed to bind!");
++index;
}
// Bind the same ones again since we looped this twice when making the query above
for (const auto& path : searchPaths)
{
[[maybe_unused]] int res = sqlite3_bind_text(statement, index, path.c_str(), static_cast<int>(path.size()), nullptr);
AZ_Assert(res == SQLITE_OK, "Statement::BindValueText: failed to bind!");
++index;
}
});
transaction.Commit();
return result;
}
@ -2876,6 +2843,46 @@ namespace AzToolsFramework
return GetResult(callName, statement, handler);
}
bool GetProductDependencyAndPathResult(
[[maybe_unused]] const char* callName,
Statement* statement,
AssetDatabaseConnection::productDependencyAndPathHandler handler)
{
Statement::SqlStatus result = statement->Step();
ProductDependencyDatabaseEntry productDependency;
AZStd::string relativeSearchPath;
auto boundColumns = CombineColumns(productDependency.GetColumns(), MakeColumns(MakeColumn("search", relativeSearchPath)));
bool validResult = result == Statement::SqlDone;
while (result == Statement::SqlOK)
{
if (!boundColumns.Fetch(statement))
{
return false;
}
if (handler(productDependency, relativeSearchPath))
{
result = statement->Step();
}
else
{
result = Statement::SqlDone;
}
validResult = true;
}
if (result == Statement::SqlError)
{
AZ_Warning(LOG_NAME, false, "Error occurred while stepping %s", callName);
return false;
}
return validResult;
}
bool GetMissingProductDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::missingProductDependencyHandler handler)
{
return GetResult(callName, statement, handler);

2
Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.h
Unescape Escape View File

@ -614,6 +614,7 @@ namespace AzToolsFramework
//! Returns any unresolved dependencies which match (by exact or wildcard match) the input searchPaths
//! The extra path returned for each row is the searchPath entry that was matched with the returned dependency entry
//! @param searchPaths vector of relative paths to search for matches
bool QueryProductDependenciesUnresolvedAdvanced(const AZStd::vector<AZStd::string>& searchPaths, productDependencyAndPathHandler handler);
bool QueryMissingProductDependencyByProductId(AZ::s64 productId, missingProductDependencyHandler handler);
@ -668,6 +669,7 @@ namespace AzToolsFramework
bool GetProductResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::productHandler handler, AZ::Uuid builderGuid = AZ::Uuid::CreateNull(), const char* jobKey = nullptr, AssetSystem::JobStatus status = AssetSystem::JobStatus::Any);
bool GetLegacySubIDsResult(const char* callname, SQLite::Statement* statement, AssetDatabaseConnection::legacySubIDsHandler handler);
bool GetProductDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::productDependencyHandler handler);
bool GetProductDependencyAndPathResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::productDependencyAndPathHandler handler);
bool GetMissingProductDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::missingProductDependencyHandler handler);
bool GetCombinedDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::combinedProductDependencyHandler handler);
bool GetFileResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::fileHandler handler);

7
Code/Framework/AzToolsFramework/AzToolsFramework/SQLite/SQLiteConnection.cpp
Unescape Escape View File

@ -302,7 +302,10 @@ namespace AzToolsFramework
return false;
}
bindCallback(statement);
if (bindCallback)
{
bindCallback(statement);
}
res = sqlite3_step(statement);
bool validResult = res == SQLITE_DONE;
@ -495,7 +498,7 @@ namespace AzToolsFramework
int res = sqlite3_prepare_v2(db, m_parentPrototype->GetSqlText().c_str(), (int)m_parentPrototype->GetSqlText().length() + 1, &m_statement, NULL);
AZ_Error("SQLiteConnection", res == SQLITE_OK, "Statement::PrepareFirstTime: failed! %s ( prototype is '%s'). Error code returned is %d.", sqlite3_errmsg(db), m_parentPrototype->GetSqlText().c_str(), res);
AZ_Assert(res == SQLITE_OK, "Statement::PrepareFirstTime: failed! %s ( prototype is '%s'). Error code returned is %d.", sqlite3_errmsg(db), m_parentPrototype->GetSqlText().c_str(), res);
return ((res == SQLITE_OK)&&(m_statement));
}

57
Code/Tools/AssetProcessor/native/tests/assetdatabase/AssetDatabaseTest.cpp
Unescape Escape View File

@ -2253,6 +2253,63 @@ namespace UnitTests
EXPECT_EQ(m_errorAbsorber->m_numAssertsAbsorbed, 0);
}
TEST_F(AssetDatabaseTest, QueryProductDependenciesUnresolvedAdvanced_HandlesLargeSearch_Success)
{
CreateCoverageTestData();
constexpr int NumTestPaths = 10000;
AZStd::vector<AZStd::string> searchPaths;
searchPaths.reserve(NumTestPaths);
for (int i = 0; i < NumTestPaths; ++i)
{
searchPaths.emplace_back(AZStd::string::format("%d.txt", i));
}
ProductDependencyDatabaseEntry dependency1(m_data->m_product1.m_productID, AZ::Uuid::CreateNull(), 0, 0, "pc", false, "*.txt");
ProductDependencyDatabaseEntry dependency2(
m_data->m_product1.m_productID, AZ::Uuid::CreateNull(), 0, 0, "pc", false, "default.xml");
m_data->m_connection.SetProductDependency(dependency1);
m_data->m_connection.SetProductDependency(dependency2);
AZStd::vector<AZStd::string> matches;
matches.reserve(NumTestPaths);
ASSERT_TRUE(m_data->m_connection.QueryProductDependenciesUnresolvedAdvanced(
searchPaths,
[&matches](AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& /*entry*/, const AZStd::string& path)
{
matches.push_back(path);
return true;
}));
ASSERT_EQ(matches.size(), searchPaths.size());
// Check the first few results match
for (int i = 0; i < 10 && i < NumTestPaths; ++i)
{
ASSERT_STREQ(matches[i].c_str(), searchPaths[i].c_str());
}
matches.clear();
searchPaths.clear();
searchPaths.push_back("default.xml");
// Run the query again to make sure a) we can b) we don't get any extra results and c) we can query for exact (non wildcard) matches
ASSERT_TRUE(m_data->m_connection.QueryProductDependenciesUnresolvedAdvanced(
searchPaths,
[&matches](AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& /*entry*/, const AZStd::string& path)
{
matches.push_back(path);
return true;
}));
ASSERT_THAT(matches, testing::ElementsAreArray(searchPaths));
}
TEST_F(AssetDatabaseTest, QueryCombined_Succeeds)
{
// This test specifically checks that the legacy subIds returned by QueryCombined are correctly matched to only the one product that they're associated with

Write Preview
Loading…
Cancel
Save