AP: product dependency optimization (#6619)

* Initial pass at optimizing product path dependency resolution Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add version of StripAssetPlatform that doesn't allocate or copy strings. Re-add missing test and fix up compile errors. Add benchmark test Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Change UpdateProductDependencies to directly call s_InsertProductDependencyQuery.BindAndStep Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add test for same filename on multiple platforms Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Rework search logic to keep track of the source of a search path (source vs product) and keep track of which search matches which dependency to avoid doing another search through every product later on Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Clean up code, expand test Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix paths not being lowercased by SanitizeForDatabase. Fix UpdateProductDependencies not updating existing dependencies Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add test for duplicate dependency matches. Fix saving duplicates Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Clean up code Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Separate test into test and benchmark versions Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Cleanup include Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix includes, switch hardcoded job manager setup to use JobManagerComponent instead Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Replaced wildcard_match with PathView::Match. Changed StripAssetPlatformNoCopy to use TokenizeNext. Removed Environment Create/Destroy calls. Made ScopedAllocatorFixture a base class of ScopedAllocatorSetupFixture Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add AZ Environment create/destroy on AP test environment Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Add missing asserts on database functions Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix incorrect usage of StripAssetPlatformNoCopy Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix source/product dependency type being ignored. Removed need for unordered_set for list of resolved dependencies. Updated unit tests Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Better variable names Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Remove testing code Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com> * Fix missing includes and namespaces Signed-off-by: amzn-mike <80125227+amzn-mike@users.noreply.github.com>
2022-01-13 08:56:24 -06:00
parent 6d1a2382e8
commit fed1278fe6
14 changed files with 742 additions and 237 deletions
@@ -51,7 +51,7 @@ namespace AssetProcessor
            case AZ::SettingsRegistryInterface::VisitAction::Begin:
            {
                // Only continue traversal if the path is exactly the AssetProcessorSettingsKey (which indicates the start of traversal)
-                // or if a "Platform *" object and it's children are being traversed 
+                // or if a "Platform *" object and it's children are being traversed
                if (jsonPath == AssetProcessorSettingsKey)
                {
                    return AZ::SettingsRegistryInterface::VisitResponse::Continue;
@@ -631,7 +631,7 @@ namespace AssetProcessor
        for (const AssetBuilderSDK::PlatformInfo& platform : m_enabledPlatforms)
        {
            AZStd::string_view currentRCParams = assetRecognizer.m_defaultParams;
-            // The "/Amazon/AssetProcessor/Settings/RC */<platform>" entry will be queried 
+            // The "/Amazon/AssetProcessor/Settings/RC */<platform>" entry will be queried
            AZ::IO::Path overrideParamsKey = AZ::IO::Path(AZ::IO::PosixPathSeparator);
            overrideParamsKey /= path;
            overrideParamsKey /= platform.m_identifier;
@@ -644,7 +644,7 @@ namespace AssetProcessor
            }
            else
            {
-                // otherwise check for tags associated with the platform 
+                // otherwise check for tags associated with the platform
                for (const AZStd::string& tag : platform.m_tags)
                {
                    overrideParamsKey.ReplaceFilename(AZ::IO::PathView(tag));
@@ -1416,6 +1416,8 @@ namespace AssetProcessor
            return QString();
        }

+        auto* fileStateInterface = AZ::Interface<AssetProcessor::IFileStateRequests>::Get();
+
        for (int pathIdx = 0; pathIdx < m_scanFolders.size(); ++pathIdx)
        {
            AssetProcessor::ScanFolderInfo scanFolderInfo = m_scanFolders[pathIdx];
@@ -1430,7 +1432,7 @@ namespace AssetProcessor
            QDir rooted(scanFolderInfo.ScanPath());
            QString absolutePath = rooted.absoluteFilePath(tempRelativeName);
            AssetProcessor::FileStateInfo fileStateInfo;
-            auto* fileStateInterface = AZ::Interface<AssetProcessor::IFileStateRequests>::Get();
+
            if (fileStateInterface)
            {
                if (fileStateInterface->GetFileInfo(absolutePath, &fileStateInfo))
@@ -1499,7 +1501,7 @@ namespace AssetProcessor
        QRegExp nameMatch{ posixRelativeName, Qt::CaseInsensitive, QRegExp::Wildcard };
        AZStd::stack<QString> dirs;
        dirs.push(sourceFolderDir.absolutePath());
-        
+
        while (!dirs.empty())
        {
            QString absolutePath = dirs.top();
@@ -1528,7 +1530,7 @@ namespace AssetProcessor
                        continue;
                    }
                }
-                
+
                QString pathMatch{ sourceFolderDir.relativeFilePath(dirIterator.filePath()) };
                if (nameMatch.exactMatch(pathMatch))
                {
@@ -131,7 +131,7 @@ namespace AssetUtilsInternal
                }
            }
        } while (!timer.hasExpired(waitTimeInSeconds * 1000)); //We will keep retrying until the timer has expired the inputted timeout
-        
+
        // once we're done, regardless of success or failure, we 'unlock' those files for further process.
        // if we failed, also re-trigger them to rebuild (the bool param at the end of the ebus call)
        QString normalized = AssetUtilities::NormalizeFilePath(outputFile);
@@ -870,23 +870,27 @@ namespace AssetUtilities
        return true;
    }

-    QString StripAssetPlatform(AZStd::string_view relativeProductPath)
+    AZStd::string_view StripAssetPlatformNoCopy(AZStd::string_view relativeProductPath)
    {
        // Skip over the assetPlatform path segment if it is matches one of the platform defaults
        // Otherwise return the path unchanged
-        AZStd::string_view strippedProductPath{ relativeProductPath };
-        if (AZStd::optional pathSegment = AZ::StringFunc::TokenizeNext(strippedProductPath, AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR);
-            pathSegment.has_value())
+
+        AZStd::string_view originalPath = relativeProductPath;
+        AZStd::optional firstPathSegment = AZ::StringFunc::TokenizeNext(relativeProductPath, AZ_CORRECT_AND_WRONG_FILESYSTEM_SEPARATOR);
+
+        if (firstPathSegment && AzFramework::PlatformHelper::GetPlatformIdFromName(*firstPathSegment) != AzFramework::PlatformId::Invalid)
        {
-            AZ::IO::FixedMaxPathString assetPlatformSegmentLower{ *pathSegment };
-            AZStd::to_lower(assetPlatformSegmentLower.begin(), assetPlatformSegmentLower.end());
-            if (AzFramework::PlatformHelper::GetPlatformIdFromName(assetPlatformSegmentLower) != AzFramework::PlatformId::Invalid)
-            {
-                return QString::fromUtf8(strippedProductPath.data(), aznumeric_cast<int>(strippedProductPath.size()));
-            }
+            return relativeProductPath;
        }

-        return QString::fromUtf8(relativeProductPath.data(), aznumeric_cast<int>(relativeProductPath.size()));
+        return originalPath;
+    }
+
+    QString StripAssetPlatform(AZStd::string_view relativeProductPath)
+    {
+        AZStd::string_view result = StripAssetPlatformNoCopy(relativeProductPath);
+
+        return QString::fromUtf8(result.data(), aznumeric_cast<int>(result.size()));
    }

    QString NormalizeFilePath(const QString& filePath)
@@ -145,7 +145,7 @@ namespace AssetUtilities
    //! Strips the first "asset platform" from the first path segment of a relative product path
    //! This is meant for removing the asset platform for paths such as "pc/MyAssetFolder/MyAsset.asset"
    //! Therefore the result here becomes "MyAssetFolder/MyAsset"
-    //! 
+    //!
    //! Similarly invoking this function on relative path that begins with the "server" platform
    //! "server/AssetFolder/Server.asset2" -> "AssetFolder/Server.asset2"
    //! This function does not strip an asset platform from anywhere, but the first path segment
@@ -153,6 +153,10 @@ namespace AssetUtilities
    //! would return a copy of the relative path
    QString StripAssetPlatform(AZStd::string_view relativeProductPath);

+    //! Same as StripAssetPlatform, but does not perform any string copies
+    //! The return result is only valid for as long as the original input is valid
+    AZStd::string_view StripAssetPlatformNoCopy(AZStd::string_view relativeProductPath);
+
    //! Converts all slashes to forward slashes, removes double slashes,
    //! replaces all indirections such as '.' or '..' as appropriate.
    //! On windows, the drive letter (if present) is converted to uppercase.