diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.cpp index 4485b0fc5e..3f3f028e1b 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.cpp @@ -850,6 +850,32 @@ namespace AzToolsFramework "SELECT * FROM ProductDependencies WHERE UnresolvedPath LIKE \":%\""; static const auto s_queryProductDependencyExclusions = MakeSqlQuery(GET_PRODUCT_DEPENDENCY_EXCLUSIONS, GET_PRODUCT_DEPENDENCY_EXCLUSIONS_STATEMENT, LOG_NAME); + static const char* CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE = + "AssetProcessor::CreateUnresolvedProductDependenciesTempTable"; + static const char* CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE_STATEMENT = + "CREATE TEMPORARY TABLE QueryProductDependenciesUnresolvedAdvanced(search)"; + static const auto s_createUnresolvedProductDependenciesTempTable = MakeSqlQuery( + CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE, CREATE_UNRESOLVED_PRODUCT_DEPENDENCIES_TEMP_TABLE_STATEMENT, LOG_NAME); + + static const char* INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES = "AssetProcessor::InsertProductDependencyTempTableValues"; + static const char* INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES_STATEMENT = + "INSERT INTO QueryProductDependenciesUnresolvedAdvanced VALUES (:filename)"; + static const auto s_queryInsertProductDependencyTempTableValues = MakeSqlQuery( + INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES, + INSERT_PRODUCT_DEPENDENCY_TEMP_TABLE_VALUES_STATEMENT, + LOG_NAME, + SqlParam(":filename")); + + static const char* GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE = + "AssetProcessor::GetUnresolvedProductDependenciesUsingTempTable"; + static const char* GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE_STATEMENT = + "SELECT * FROM ProductDependencies INNER JOIN QueryProductDependenciesUnresolvedAdvanced " + "ON (UnresolvedPath LIKE \"%*%\" AND search LIKE REPLACE(UnresolvedPath, \"*\", \"%\")) OR search = UnresolvedPath"; + static const auto s_queryGetUnresolvedProductDependenciesUsingTempTable = MakeSqlQuery( + GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE, + GET_UNRESOLVED_PRODUCT_DEPENDENCIES_USING_TEMP_TABLE_STATEMENT, + LOG_NAME); + // lookup by primary key static const char* QUERY_FILE_BY_FILEID = "AzToolsFramework::AssetDatabase::QueryFileByFileID"; static const char* QUERY_FILE_BY_FILEID_STATEMENT = @@ -1814,6 +1840,9 @@ namespace AzToolsFramework AddStatement(m_databaseConnection, s_queryAllProductdependencies); AddStatement(m_databaseConnection, s_queryUnresolvedProductDependencies); AddStatement(m_databaseConnection, s_queryProductDependencyExclusions); + AddStatement(m_databaseConnection, s_createUnresolvedProductDependenciesTempTable); + AddStatement(m_databaseConnection, s_queryInsertProductDependencyTempTableValues); + AddStatement(m_databaseConnection, s_queryGetUnresolvedProductDependenciesUsingTempTable); AddStatement(m_databaseConnection, s_queryFileByFileid); AddStatement(m_databaseConnection, s_queryFilesByFileName); @@ -2480,88 +2509,26 @@ namespace AzToolsFramework return s_queryProductDependencyExclusions.BindAndQuery(*m_databaseConnection, handler, &GetProductDependencyResult); } - bool AssetDatabaseConnection::QueryProductDependenciesUnresolvedAdvanced(const AZStd::vector& searchPaths, productDependencyAndPathHandler handler) + bool AssetDatabaseConnection::QueryProductDependenciesUnresolvedAdvanced( + const AZStd::vector& searchPaths, productDependencyAndPathHandler handler) { - AZStd::string sql = R"END(select c.*, b.search FROM (select REPLACE(UnresolvedPath, "*", "%") as search, ProductDependencyID from ProductDependencies where UnresolvedPath LIKE "%*%") as a, ()END"; + ScopedTransaction transaction(m_databaseConnection); - bool first = true; + bool result = s_createUnresolvedProductDependenciesTempTable.BindAndStep(*m_databaseConnection); - for(int i = 0; i < searchPaths.size(); ++i) + for (auto&& path : searchPaths) { - if(first) - { - sql += "SELECT ? as search "; - first = false; - } - else - { - sql += "UNION SELECT ? "; - } - } - - sql += R"END() as b - INNER JOIN ProductDependencies as c ON c.ProductDependencyID = a.ProductDependencyID - WHERE b.search LIKE a.search - UNION SELECT p.*, UnresolvedPath - FROM ProductDependencies as p - WHERE UnresolvedPath != "" - )END"; - - first = true; - - for (int i = 0; i < searchPaths.size(); ++i) - { - if(first) - { - sql += " AND "; - first = false; - } - else - { - sql += " OR "; - } - - sql += "UnresolvedPath = ?"; + result = s_queryInsertProductDependencyTempTableValues.BindAndStep(*m_databaseConnection, path.c_str()) && result; } - bool result = m_databaseConnection->ExecuteRawSqlQuery(sql, [handler](sqlite3_stmt* statement) - { - ProductDependencyDatabaseEntry entry; - - entry.m_productDependencyID = SQLite::GetColumnInt64(statement, 0); - entry.m_productPK = SQLite::GetColumnInt64(statement, 1); - entry.m_dependencySourceGuid = SQLite::GetColumnUuid(statement, 2); - entry.m_dependencySubID = GetColumnInt(statement, 3); - entry.m_platform = GetColumnText(statement, 4); - entry.m_dependencyFlags = GetColumnInt64(statement, 5); - entry.m_unresolvedPath = GetColumnText(statement, 6); - entry.m_dependencyType = static_cast(GetColumnInt(statement, 7)); - entry.m_fromAssetId = sqlite3_column_int(statement, 8); - - AZStd::string matchedPath = GetColumnText(statement, 9); + result = s_queryGetUnresolvedProductDependenciesUsingTempTable.BindAndQuery( + *m_databaseConnection, handler, &GetProductDependencyAndPathResult) && + result; - handler(entry, matchedPath); + result = m_databaseConnection->ExecuteRawSqlQuery("DROP TABLE QueryProductDependenciesUnresolvedAdvanced", nullptr, nullptr) && + result; - return true; - }, [&searchPaths](sqlite3_stmt* statement) - { - int index = 1; - - for (const auto& path : searchPaths) - { - [[maybe_unused]] int res = sqlite3_bind_text(statement, index, path.c_str(), static_cast(path.size()), nullptr); - AZ_Assert(res == SQLITE_OK, "Statement::BindValueText: failed to bind!"); - ++index; - } - - // Bind the same ones again since we looped this twice when making the query above - for (const auto& path : searchPaths) - { - [[maybe_unused]] int res = sqlite3_bind_text(statement, index, path.c_str(), static_cast(path.size()), nullptr); - AZ_Assert(res == SQLITE_OK, "Statement::BindValueText: failed to bind!"); - ++index; - } - }); + transaction.Commit(); return result; } @@ -2876,6 +2843,46 @@ namespace AzToolsFramework return GetResult(callName, statement, handler); } + bool GetProductDependencyAndPathResult( + [[maybe_unused]] const char* callName, + Statement* statement, + AssetDatabaseConnection::productDependencyAndPathHandler handler) + { + Statement::SqlStatus result = statement->Step(); + + ProductDependencyDatabaseEntry productDependency; + + AZStd::string relativeSearchPath; + auto boundColumns = CombineColumns(productDependency.GetColumns(), MakeColumns(MakeColumn("search", relativeSearchPath))); + + bool validResult = result == Statement::SqlDone; + while (result == Statement::SqlOK) + { + if (!boundColumns.Fetch(statement)) + { + return false; + } + + if (handler(productDependency, relativeSearchPath)) + { + result = statement->Step(); + } + else + { + result = Statement::SqlDone; + } + validResult = true; + } + + if (result == Statement::SqlError) + { + AZ_Warning(LOG_NAME, false, "Error occurred while stepping %s", callName); + return false; + } + + return validResult; + } + bool GetMissingProductDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::missingProductDependencyHandler handler) { return GetResult(callName, statement, handler); diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.h b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.h index e7e7522908..6e10752997 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetDatabase/AssetDatabaseConnection.h @@ -614,6 +614,7 @@ namespace AzToolsFramework //! Returns any unresolved dependencies which match (by exact or wildcard match) the input searchPaths //! The extra path returned for each row is the searchPath entry that was matched with the returned dependency entry + //! @param searchPaths vector of relative paths to search for matches bool QueryProductDependenciesUnresolvedAdvanced(const AZStd::vector& searchPaths, productDependencyAndPathHandler handler); bool QueryMissingProductDependencyByProductId(AZ::s64 productId, missingProductDependencyHandler handler); @@ -668,6 +669,7 @@ namespace AzToolsFramework bool GetProductResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::productHandler handler, AZ::Uuid builderGuid = AZ::Uuid::CreateNull(), const char* jobKey = nullptr, AssetSystem::JobStatus status = AssetSystem::JobStatus::Any); bool GetLegacySubIDsResult(const char* callname, SQLite::Statement* statement, AssetDatabaseConnection::legacySubIDsHandler handler); bool GetProductDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::productDependencyHandler handler); + bool GetProductDependencyAndPathResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::productDependencyAndPathHandler handler); bool GetMissingProductDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::missingProductDependencyHandler handler); bool GetCombinedDependencyResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::combinedProductDependencyHandler handler); bool GetFileResult(const char* callName, SQLite::Statement* statement, AssetDatabaseConnection::fileHandler handler); diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/SQLite/SQLiteConnection.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/SQLite/SQLiteConnection.cpp index ee2534ed9c..b738de75a9 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/SQLite/SQLiteConnection.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/SQLite/SQLiteConnection.cpp @@ -302,7 +302,10 @@ namespace AzToolsFramework return false; } - bindCallback(statement); + if (bindCallback) + { + bindCallback(statement); + } res = sqlite3_step(statement); bool validResult = res == SQLITE_DONE; @@ -495,7 +498,7 @@ namespace AzToolsFramework int res = sqlite3_prepare_v2(db, m_parentPrototype->GetSqlText().c_str(), (int)m_parentPrototype->GetSqlText().length() + 1, &m_statement, NULL); - AZ_Error("SQLiteConnection", res == SQLITE_OK, "Statement::PrepareFirstTime: failed! %s ( prototype is '%s'). Error code returned is %d.", sqlite3_errmsg(db), m_parentPrototype->GetSqlText().c_str(), res); + AZ_Assert(res == SQLITE_OK, "Statement::PrepareFirstTime: failed! %s ( prototype is '%s'). Error code returned is %d.", sqlite3_errmsg(db), m_parentPrototype->GetSqlText().c_str(), res); return ((res == SQLITE_OK)&&(m_statement)); } diff --git a/Code/Tools/AssetProcessor/native/tests/assetdatabase/AssetDatabaseTest.cpp b/Code/Tools/AssetProcessor/native/tests/assetdatabase/AssetDatabaseTest.cpp index 1a9779f853..8bffdc2f09 100644 --- a/Code/Tools/AssetProcessor/native/tests/assetdatabase/AssetDatabaseTest.cpp +++ b/Code/Tools/AssetProcessor/native/tests/assetdatabase/AssetDatabaseTest.cpp @@ -2253,6 +2253,63 @@ namespace UnitTests EXPECT_EQ(m_errorAbsorber->m_numAssertsAbsorbed, 0); } + TEST_F(AssetDatabaseTest, QueryProductDependenciesUnresolvedAdvanced_HandlesLargeSearch_Success) + { + CreateCoverageTestData(); + + constexpr int NumTestPaths = 10000; + + AZStd::vector searchPaths; + + searchPaths.reserve(NumTestPaths); + + for (int i = 0; i < NumTestPaths; ++i) + { + searchPaths.emplace_back(AZStd::string::format("%d.txt", i)); + } + + ProductDependencyDatabaseEntry dependency1(m_data->m_product1.m_productID, AZ::Uuid::CreateNull(), 0, 0, "pc", false, "*.txt"); + ProductDependencyDatabaseEntry dependency2( + m_data->m_product1.m_productID, AZ::Uuid::CreateNull(), 0, 0, "pc", false, "default.xml"); + + m_data->m_connection.SetProductDependency(dependency1); + m_data->m_connection.SetProductDependency(dependency2); + + AZStd::vector matches; + matches.reserve(NumTestPaths); + + ASSERT_TRUE(m_data->m_connection.QueryProductDependenciesUnresolvedAdvanced( + searchPaths, + [&matches](AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& /*entry*/, const AZStd::string& path) + { + matches.push_back(path); + return true; + })); + + ASSERT_EQ(matches.size(), searchPaths.size()); + + // Check the first few results match + for (int i = 0; i < 10 && i < NumTestPaths; ++i) + { + ASSERT_STREQ(matches[i].c_str(), searchPaths[i].c_str()); + } + + matches.clear(); + searchPaths.clear(); + searchPaths.push_back("default.xml"); + + // Run the query again to make sure a) we can b) we don't get any extra results and c) we can query for exact (non wildcard) matches + ASSERT_TRUE(m_data->m_connection.QueryProductDependenciesUnresolvedAdvanced( + searchPaths, + [&matches](AzToolsFramework::AssetDatabase::ProductDependencyDatabaseEntry& /*entry*/, const AZStd::string& path) + { + matches.push_back(path); + return true; + })); + + ASSERT_THAT(matches, testing::ElementsAreArray(searchPaths)); + } + TEST_F(AssetDatabaseTest, QueryCombined_Succeeds) { // This test specifically checks that the legacy subIds returned by QueryCombined are correctly matched to only the one product that they're associated with