Improve handling of cached data limits. (#5232)

* Remove size limit on cached profile regions.

Signed-off-by: rbarrand <43485729+hershey5045@users.noreply.github.com>

* Discard excess profiling data once limit has been reached. Warn users only the first time the limit is reached.

Signed-off-by: rbarrand <43485729+hershey5045@users.noreply.github.com>

* Fix bug that was clearing cached time regions instead of the cached time regions map.

Signed-off-by: rbarrand <43485729+hershey5045@users.noreply.github.com>

Gems/Profiler/Code/Source/CpuProfilerImpl.cpp

@@ -282,7 +282,7 @@ namespace Profiler
             m_stackLevel = 0;
             m_cachedTimeRegionMap.clear();
             m_timeRegionStack.clear();
-            m_cachedTimeRegions.clear();
+            ResetCachedData();
         }
 
         timeRegion.m_stackDepth = aznumeric_cast<uint16_t>(m_stackLevel);
@@ -329,8 +329,21 @@ namespace Profiler
         {
             return;
         }
-        // Add an entry to the cached region
+        // Add an entry to the cached region. Discard excess data in case there is too much to handle.
-        m_cachedTimeRegions.push_back(timeRegionCached);
+        if (m_cachedTimeRegions.size() < TimeRegionStackSize)
+        {
+            m_cachedTimeRegions.push_back(timeRegionCached);
+        }
+        // Warn only once per thread if the cached data limit has been reached.
+        else if (!m_cachedDataLimitReached)
+        {
+            AZ_Warning(
+                "Profiler", false,
+                "Limit for profiling data has been reached by thread %i. Excess data will be discarded. Considering moving or reducing "
+                "profiler markers to prevent data loss.",
+                m_executingThreadId);
+            m_cachedDataLimitReached = true;
+        }
 
         // If the stack is empty, add it to the local cache map. Only gets called when the stack is empty
         // NOTE: this is where the largest overhead will be, but due to it only being called when the stack is empty
@@ -354,7 +367,7 @@ namespace Profiler
             }
 
             // Clear the cached regions
-            m_cachedTimeRegions.clear();
+            ResetCachedData();
         }
     }
 
@@ -371,10 +384,17 @@ namespace Profiler
                 m_cachedTimeRegionMap.clear();
                 m_hitSizeLimitMap.clear();
             }
+
             m_cachedTimeRegionMutex.unlock();
         }
     }
 
+    void CpuTimingLocalStorage::ResetCachedData()
+    {
+        m_cachedTimeRegions.clear();
+        m_cachedDataLimitReached = false;
+    }
+
     // --- CpuProfilingStatisticsSerializer ---
 
     CpuProfilingStatisticsSerializer::CpuProfilingStatisticsSerializer(const AZStd::ring_buffer<CpuProfiler::TimeRegionMap>& continuousData)

Gems/Profiler/Code/Source/CpuProfilerImpl.h

@@ -50,6 +50,9 @@ namespace Profiler
         // Tries to flush the map to the passed parameter, only if the thread's mutex is unlocked
         void TryFlushCachedMap(CpuProfiler::ThreadTimeRegionMap& cachedRegionMap);
 
+        // Clears m_cachedTimeRegions and resets m_cachedDataLimitReached flag.
+        void ResetCachedData();
+
         AZStd::thread_id m_executingThreadId;
         // Keeps track of the current thread's stack depth
         uint32_t m_stackLevel = 0u;
@@ -75,6 +78,9 @@ namespace Profiler
 
         // Keep track of the regions that have hit the size limit so we don't have to lock to check
         AZStd::map<AZStd::string, bool> m_hitSizeLimitMap;
+
+        // Keeps track of the first time cached data limit was reached.
+        bool m_cachedDataLimitReached = false;
     };
 
     //! CpuProfiler will keep track of the registered threads, and