birkeh/o3de
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Integrating github/staging through commit 5f214be

Browse Source
This commit is contained in:
alexpete
2021-04-13 17:18:57 -07:00
parent 2a9dfb7a1d
commit 8469c9ca0a
904 changed files with 9289 additions and 7157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Code/Tools/AssetProcessor/native/AssetManager/FileStateCache.cpp
+4 -1
View File
@@ -149,7 +149,10 @@ namespace AssetProcessor
{
return normalized.toLower();
}
return normalized;
else
{
return normalized;
}
}
void FileStateCache::AddOrUpdateFileInternal(QFileInfo fileInfo)
Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.cpp
+66 -32
View File
@@ -476,7 +476,9 @@ namespace AssetProcessor
// CheckDeletedSourceFile actually expects the database name as the second value
// iter.key is the full path normalized. iter.value is the database path.
// we need the relative path too, which involves removing the scan folder outputprefix it present:
CheckDeletedSourceFile(iter.key(), iter.value().m_sourceRelativeToWatchFolder, iter.value().m_sourceDatabaseName);
CheckDeletedSourceFile(
iter.key(), iter.value().m_sourceRelativeToWatchFolder, iter.value().m_sourceDatabaseName,
AZStd::chrono::system_clock::now());
}
// we want to remove any left over scan folders from the database only after
@@ -1524,7 +1526,7 @@ namespace AssetProcessor
// even if the entry already exists,
// overwrite the entry here, so if you modify, then delete it, its the latest action thats always on the list.
m_filesToExamine[normalizedFilePath] = FileEntry(normalizedFilePath, source.m_isDelete, source.m_isFromScanner);
m_filesToExamine[normalizedFilePath] = FileEntry(normalizedFilePath, source.m_isDelete, source.m_isFromScanner, source.m_initialProcessTime);
// this block of code adds anything which DEPENDS ON the file that was changed, back into the queue so that files
// that depend on it also re-analyze in case they need rebuilding. However, files that are deleted will be added
@@ -1773,40 +1775,31 @@ namespace AssetProcessor
return successfullyRemoved;
}
void AssetProcessorManager::CheckDeletedSourceFile(QString normalizedPath, QString relativePath, QString databaseSourceFile)
void AssetProcessorManager::CheckDeletedSourceFile(QString normalizedPath, QString relativePath, QString databaseSourceFile,
AZStd::chrono::system_clock::time_point initialProcessTime)
{
// getting here means an input asset has been deleted
// and no overrides exist for it.
// we must delete its products.
using namespace AzToolsFramework::AssetDatabase;
// If we fail to delete a product, the deletion event gets requeued
// To avoid retrying forever, we keep track of the time of the first deletion failure and only retry
// if less than this amount of time has passed.
constexpr int MaxRetryPeriodMS = 500;
AZStd::chrono::duration<double, AZStd::milli> duration = AZStd::chrono::system_clock::now() - initialProcessTime;
// Check if this file causes any file types to be re-evaluated
CheckMetaDataRealFiles(normalizedPath);
// when a source is deleted, we also have to queue anything that depended on it, for re-processing:
SourceFileDependencyEntryContainer results;
m_stateData->GetSourceFileDependenciesByDependsOnSource(databaseSourceFile, SourceFileDependencyEntry::DEP_Any, results);
// the jobIdentifiers that have identified it as a job dependency
for (SourceFileDependencyEntry& existingEntry : results)
if (initialProcessTime > AZStd::chrono::system_clock::time_point{}
&& duration >= AZStd::chrono::milliseconds(MaxRetryPeriodMS))
{
// this row is [Source] --> [Depends on Source].
QString absolutePath = m_platformConfig->FindFirstMatchingFile(QString::fromUtf8(existingEntry.m_source.c_str()));
if (!absolutePath.isEmpty())
{
AssessFileInternal(absolutePath, false);
}
// also, update it in the database to be missing, ie, add the "missing file" prefix:
existingEntry.m_dependsOnSource = QString(PlaceHolderFileName + relativePath).toUtf8().constData();
m_stateData->RemoveSourceFileDependency(existingEntry.m_sourceDependencyID);
m_stateData->SetSourceFileDependency(existingEntry);
AZ_Warning(AssetProcessor::ConsoleChannel, false, "Failed to delete product(s) from source file `%s` after retrying for %fms. Giving up.",
normalizedPath.toUtf8().constData(), duration.count());
return;
}
// now that the right hand column (in terms of [thing] -> [depends on thing]) has been updated, eliminate anywhere its on the left hand side:
results.clear();
m_stateData->GetDependsOnSourceBySource(databaseSourceFile.toUtf8().constData(), SourceFileDependencyEntry::DEP_Any, results);
m_stateData->RemoveSourceFileDependencies(results);
bool deleteFailure = false;
AzToolsFramework::AssetDatabase::SourceDatabaseEntryContainer sources;
if (m_stateData->GetSourcesBySourceName(databaseSourceFile, sources))
{
for (const auto& source : sources)
@@ -1827,7 +1820,13 @@ namespace AssetProcessor
{
// DeleteProducts will make an attempt to retry deleting each product
// We can't just re-queue the whole file with CheckSource because we're deleting bits from the database as we go
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Delete failed on %s.\n", normalizedPath.toUtf8().constData());
deleteFailure = true;
CheckSource(FileEntry(
normalizedPath, true, false,
initialProcessTime > AZStd::chrono::system_clock::time_point{} ? initialProcessTime
: AZStd::chrono::system_clock::now()));
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "Delete failed on %s. Will retry!\n", normalizedPath.toUtf8().constData());
continue;
}
}
else
@@ -1843,13 +1842,48 @@ namespace AssetProcessor
Q_EMIT JobRemoved(jobInfo);
}
}
// delete the source from the database too since otherwise it believes we have no products.
m_stateData->RemoveSource(source.m_sourceID);
if (!deleteFailure)
{
// delete the source from the database too since otherwise it believes we have no products.
m_stateData->RemoveSource(source.m_sourceID);
}
}
}
if(deleteFailure)
{
return;
}
// Check if this file causes any file types to be re-evaluated
CheckMetaDataRealFiles(normalizedPath);
Q_EMIT SourceDeleted(databaseSourceFile); // note that this removes it from the RC Queue Model, also
// when a source is deleted, we also have to queue anything that depended on it, for re-processing:
SourceFileDependencyEntryContainer results;
m_stateData->GetSourceFileDependenciesByDependsOnSource(databaseSourceFile, SourceFileDependencyEntry::DEP_Any, results);
// the jobIdentifiers that have identified it as a job dependency
for (SourceFileDependencyEntry& existingEntry : results)
{
// this row is [Source] --> [Depends on Source].
QString absolutePath = m_platformConfig->FindFirstMatchingFile(QString::fromUtf8(existingEntry.m_source.c_str()));
if (!absolutePath.isEmpty())
{
AssessFileInternal(absolutePath, false);
}
// also, update it in the database to be missing, ie, add the "missing file" prefix:
existingEntry.m_dependsOnSource = QString(PlaceHolderFileName + relativePath).toUtf8().constData();
m_stateData->RemoveSourceFileDependency(existingEntry.m_sourceDependencyID);
m_stateData->SetSourceFileDependency(existingEntry);
}
// now that the right hand column (in terms of [thing] -> [depends on thing]) has been updated, eliminate anywhere its on the left
// hand side:
results.clear();
m_stateData->GetDependsOnSourceBySource(databaseSourceFile.toUtf8().constData(), SourceFileDependencyEntry::DEP_Any, results);
m_stateData->RemoveSourceFileDependencies(results);
Q_EMIT SourceDeleted(databaseSourceFile); // note that this removes it from the RC Queue Model, also
}
void AssetProcessorManager::AddKnownFoldersRecursivelyForFile(QString fullFile, QString root)
@@ -2568,7 +2602,7 @@ namespace AssetProcessor
jobdetail.m_jobParam[AZ_CRC(AutoFailReasonKey)] = AZStd::string::format(
"Source file ( %s ) contains non ASCII characters.\n"
"Lumberyard currently only supports file paths having ASCII characters and therefore asset processor will not be able to process this file.\n"
"Open 3D Engine currently only supports file paths having ASCII characters and therefore asset processor will not be able to process this file.\n"
"Please rename the source file to fix this error.\n",
normalizedPath.toUtf8().data());
@@ -2641,7 +2675,7 @@ namespace AssetProcessor
AZ::Uuid sourceUUID = AssetUtilities::CreateSafeSourceUUIDFromName(databasePathToFile.toUtf8().data());
AzToolsFramework::AssetSystem::SourceFileNotificationMessage message(AZ::OSString(sourceFile.toUtf8().constData()), AZ::OSString(scanFolderInfo->ScanPath().toUtf8().constData()), AzToolsFramework::AssetSystem::SourceFileNotificationMessage::FileRemoved, sourceUUID);
EBUS_EVENT(AssetProcessor::ConnectionBus, Send, 0, message);
CheckDeletedSourceFile(normalizedPath, relativePathToFile, databasePathToFile);
CheckDeletedSourceFile(normalizedPath, relativePathToFile, databasePathToFile, examineFile.m_initialProcessTime);
}
else
{
Code/Tools/AssetProcessor/native/AssetManager/assetProcessorManager.h
+6 -2
View File
@@ -116,13 +116,15 @@ namespace AssetProcessor
QString m_fileName;
bool m_isDelete = false;
bool m_isFromScanner = false;
AZStd::chrono::system_clock::time_point m_initialProcessTime{};
FileEntry() = default;
FileEntry(const QString& fileName, bool isDelete, bool isFromScanner=false)
FileEntry(const QString& fileName, bool isDelete, bool isFromScanner = false, AZStd::chrono::system_clock::time_point initialProcessTime = {})
: m_fileName(fileName)
, m_isDelete(isDelete)
, m_isFromScanner(isFromScanner)
, m_initialProcessTime(initialProcessTime)
{
}
@@ -305,7 +307,9 @@ namespace AssetProcessor
void CheckSource(const FileEntry& source);
void CheckMissingJobs(QString relativeSourceFile, const ScanFolderInfo* scanFolder, const AZStd::vector<JobDetails>& jobsThisTime);
void CheckDeletedProductFile(QString normalizedPath);
void CheckDeletedSourceFile(QString normalizedPath, QString relativePath, QString databaseSourceFile);
void CheckDeletedSourceFile(
QString normalizedPath, QString relativePath, QString databaseSourceFile,
AZStd::chrono::system_clock::time_point initialProcessTime);
void CheckModifiedSourceFile(QString normalizedPath, QString databaseSourceFile, const ScanFolderInfo* scanFolderInfo);
bool AnalyzeJob(JobDetails& details);
void CheckDeletedCacheFolder(QString normalizedPath);
Code/Tools/AssetProcessor/native/resourcecompiler/RCBuilder.cpp
+5 -5
View File
@@ -424,12 +424,12 @@ namespace AssetProcessor
bool InternalRecognizerBasedBuilder::FindRC(QString& rcAbsolutePathOut)
{
char executableDirectory[AZ_MAX_PATH_LEN];
if (AZ::Utils::GetExecutableDirectory(executableDirectory, AZStd::size(executableDirectory)) == AZ::Utils::ExecutablePathResult::Success)
AZ::IO::FixedMaxPath executableDirectory = AZ::Utils::GetExecutableDirectory();
executableDirectory /= ASSETPROCESSOR_TRAIT_LEGACY_RC_RELATIVE_PATH;
if (AZ::IO::SystemFile::Exists(executableDirectory.c_str()))
{
rcAbsolutePathOut = QString("%1/%2").arg(executableDirectory).arg(QString(ASSETPROCESSOR_TRAIT_LEGACY_RC_RELATIVE_PATH));
return AZ::IO::SystemFile::Exists(rcAbsolutePathOut.toUtf8().data());
rcAbsolutePathOut = QString::fromUtf8(executableDirectory.c_str(), executableDirectory.Native().size());
return true;
}
return false;
Code/Tools/AssetProcessor/native/tests/assetdatabase/AssetDatabaseTest.cpp
+4 -4
View File
@@ -81,7 +81,7 @@ namespace UnitTests
// Product: "someproduct4.dds" subid: 4
void CreateCoverageTestData()
{
m_data->m_scanFolder = { "c:/lumberyard/dev", "dev", "rootportkey", "" };
m_data->m_scanFolder = { "c:/O3DE/dev", "dev", "rootportkey", "" };
ASSERT_TRUE(m_data->m_connection.SetScanFolder(m_data->m_scanFolder));
m_data->m_sourceFile1 = { m_data->m_scanFolder.m_scanFolderID, "somefile.tif", AZ::Uuid::CreateRandom(), "AnalysisFingerprint1"};
@@ -239,7 +239,7 @@ namespace UnitTests
// we'll create all of those first (except product) before starting the product test.
//add a scanfolder. None of this has to exist in real disk, this is a db test only.
ScanFolderDatabaseEntry scanFolder {"c:/lumberyard/dev", "dev", "rootportkey", ""};
ScanFolderDatabaseEntry scanFolder {"c:/O3DE/dev", "dev", "rootportkey", ""};
EXPECT_TRUE(m_data->m_connection.SetScanFolder(scanFolder));
ASSERT_NE(scanFolder.m_scanFolderID, AzToolsFramework::AssetDatabase::InvalidEntryId);
@@ -278,7 +278,7 @@ namespace UnitTests
// to add a product legitimately you have to have a full chain of primary keys, chain is:
// ScanFolder --> Source --> job --> product.
// we'll create all of those first (except product) before starting the product test.
ScanFolderDatabaseEntry scanFolder{ "c:/lumberyard/dev", "dev", "rootportkey", "" };
ScanFolderDatabaseEntry scanFolder{ "c:/O3DE/dev", "dev", "rootportkey", "" };
ASSERT_TRUE(m_data->m_connection.SetScanFolder(scanFolder));
SourceDatabaseEntry sourceEntry{ scanFolder.m_scanFolderID, "somefile.tif", AZ::Uuid::CreateRandom(), "fingerprint1" };
@@ -323,7 +323,7 @@ namespace UnitTests
// this is actually a very common case (same job id, same subID)
TEST_F(AssetDatabaseTest, SetProduct_SpecificPK_Succeeds_SameSubID_SameJobID)
{
ScanFolderDatabaseEntry scanFolder{ "c:/lumberyard/dev", "dev", "rootportkey", "" };
ScanFolderDatabaseEntry scanFolder{ "c:/O3DE/dev", "dev", "rootportkey", "" };
ASSERT_TRUE(m_data->m_connection.SetScanFolder(scanFolder));
SourceDatabaseEntry sourceEntry{ scanFolder.m_scanFolderID, "somefile.tif", AZ::Uuid::CreateRandom(), "fingerprint1" };
ASSERT_TRUE(m_data->m_connection.SetSource(sourceEntry));
Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.cpp
+125
View File
@@ -4540,9 +4540,17 @@ void ModtimeScanningTest::TearDown()
void ModtimeScanningTest::ProcessAssetJobs()
{
m_data->m_productPaths.clear();
for (const auto& processResult : m_data->m_processResults)
{
auto file = QDir(processResult.m_destinationPath).absoluteFilePath(processResult.m_jobEntry.m_databaseSourceName + ".arc1");
m_data->m_productPaths.emplace(
QDir(processResult.m_jobEntry.m_watchFolderPath)
.absoluteFilePath(processResult.m_jobEntry.m_databaseSourceName)
.toUtf8()
.constData(),
file);
// Create the file on disk
ASSERT_TRUE(UnitTestUtils::CreateDummyFile(file, "products."));
@@ -4793,6 +4801,123 @@ TEST_F(ModtimeScanningTest, ModtimeSkipping_ModifyFile_AndThenRevert_ProcessesAg
ExpectWork(2, 2);
}
struct LockedFileTest
: ModtimeScanningTest
, AssetProcessor::ConnectionBus::Handler
{
MOCK_METHOD3(SendRaw, size_t (unsigned, unsigned, const QByteArray&));
MOCK_METHOD3(SendPerPlatform, size_t (unsigned, const AzFramework::AssetSystem::BaseAssetProcessorMessage&, const QString&));
MOCK_METHOD4(SendRawPerPlatform, size_t (unsigned, unsigned, const QByteArray&, const QString&));
MOCK_METHOD2(SendRequest, unsigned (const AzFramework::AssetSystem::BaseAssetProcessorMessage&, const ResponseCallback&));
MOCK_METHOD2(SendResponse, size_t (unsigned, const AzFramework::AssetSystem::BaseAssetProcessorMessage&));
MOCK_METHOD1(RemoveResponseHandler, void (unsigned));
size_t Send(unsigned, const AzFramework::AssetSystem::BaseAssetProcessorMessage&) override
{
if(m_callback)
{
m_callback();
}
return 0;
}
void SetUp() override
{
ModtimeScanningTest::SetUp();
ConnectionBus::Handler::BusConnect(0);
}
void TearDown() override
{
ConnectionBus::Handler::BusDisconnect();
ModtimeScanningTest::TearDown();
}
AZStd::function<void()> m_callback;
};
TEST_F(LockedFileTest, DeleteFile_LockedProduct_DeleteFails)
{
auto theFile = m_data->m_absolutePath[1].toUtf8();
const char* theFileString = theFile.constData();
auto [sourcePath, productPath] = *m_data->m_productPaths.find(theFileString);
{
QFile file(theFileString);
file.remove();
}
ASSERT_GT(m_data->m_productPaths.size(), 0);
QFile product(productPath);
ASSERT_TRUE(product.open(QIODevice::ReadOnly));
// Check if we can delete the file now, if we can't, proceed with the test
// If we can, it means the OS running this test doesn't lock open files so there's nothing to test
if (!AZ::IO::SystemFile::Delete(productPath.toUtf8().constData()))
{
QMetaObject::invokeMethod(
m_assetProcessorManager.get(), "AssessDeletedFile", Qt::QueuedConnection, Q_ARG(QString, QString(theFileString)));
EXPECT_TRUE(BlockUntilIdle(5000));
EXPECT_TRUE(QFile::exists(productPath));
EXPECT_EQ(m_data->m_deletedSources.size(), 0);
}
else
{
SUCCEED() << "Skipping test. OS does not lock open files.";
}
}
TEST_F(LockedFileTest, DeleteFile_LockedProduct_DeletesWhenReleased)
{
auto theFile = m_data->m_absolutePath[1].toUtf8();
const char* theFileString = theFile.constData();
auto [sourcePath, productPath] = *m_data->m_productPaths.find(theFileString);
{
QFile file(theFileString);
file.remove();
}
ASSERT_GT(m_data->m_productPaths.size(), 0);
QFile product(productPath);
ASSERT_TRUE(product.open(QIODevice::ReadOnly));
// Check if we can delete the file now, if we can't, proceed with the test
// If we can, it means the OS running this test doesn't lock open files so there's nothing to test
if (!AZ::IO::SystemFile::Delete(productPath.toUtf8().constData()))
{
AZStd::thread workerThread;
m_callback = [&product, &workerThread]() {
workerThread = AZStd::thread([&product]() {
AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(60));
product.close();
});
};
QMetaObject::invokeMethod(
m_assetProcessorManager.get(), "AssessDeletedFile", Qt::QueuedConnection, Q_ARG(QString, QString(theFileString)));
EXPECT_TRUE(BlockUntilIdle(5000));
EXPECT_FALSE(QFile::exists(productPath));
EXPECT_EQ(m_data->m_deletedSources.size(), 1);
workerThread.join();
}
else
{
SUCCEED() << "Skipping test. OS does not lock open files.";
}
}
TEST_F(ModtimeScanningTest, ModtimeSkipping_ModifyFilesSameHash_BothProcess)
{
using namespace AzToolsFramework::AssetSystem;
Code/Tools/AssetProcessor/native/tests/assetmanager/AssetProcessorManagerTest.h
+1
View File
@@ -174,6 +174,7 @@ struct ModtimeScanningTest
QString m_relativePathFromWatchFolder[3];
AZStd::vector<QString> m_absolutePath;
AZStd::vector<AssetProcessor::JobDetails> m_processResults;
AZStd::unordered_multimap<AZStd::string, QString> m_productPaths;
AZStd::vector<QString> m_deletedSources;
AZStd::shared_ptr<AssetProcessor::InternalMockBuilder> m_builderTxtBuilder;
MockBuilderInfoHandler m_mockBuilderInfoHandler;
Code/Tools/AssetProcessor/native/ui/ProductAssetDetailsPanel.cpp
+1 -1
View File
@@ -112,7 +112,7 @@ namespace AssetProcessor
[&](AzToolsFramework::AssetDatabase::SourceDatabaseEntry& sourceEntry)
{
assetId = AZ::Data::AssetId(sourceEntry.m_sourceGuid, productItemData->m_databaseInfo.m_subID);
// Use a decimal value to display the sub ID and not hex. Lumberyard is not consistent about
// Use a decimal value to display the sub ID and not hex. Open 3D Engine is not consistent about
// how sub IDs are displayed, so it's important to double check what format a sub ID is in before using it elsewhere.
m_ui->productAssetIdValueLabel->setText(assetId.ToString<AZStd::string>(AZ::Data::AssetId::SubIdDisplayType::Decimal).c_str());
Code/Tools/AssetProcessor/native/ui/SourceAssetTreeModel.cpp
+1 -1
View File
@@ -76,7 +76,7 @@ namespace AssetProcessor
AzFramework::StringFunc::AssetDatabasePath::Join(scanFolder.m_scanFolder.c_str(), fullPath.c_str(), fullPath, true, false);
// It's common for Lumberyard game projects and scan folders to be in a subfolder
// It's common for Open 3D Engine game projects and scan folders to be in a subfolder
// of the engine install. To improve readability of the source files, strip out
// that portion of the path if it overlaps.
if (!m_assetRootSet)
Code/Tools/AssetProcessor/native/ui/style/AssetProcessor.qrc
+1 -1
View File
@@ -16,6 +16,6 @@
<file>AssetProcessor_arrow_down.svg</file>
<file>AssetProcessor_arrow_up.svg</file>
<file>AssetProcessor_refresh.png</file>
<file>lyassetprocessor.png</file>
<file>o3de_assetprocessor.png</file>
</qresource>
</RCC>
Code/Tools/AssetProcessor/native/ui/style/lyassetprocessor.ico
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9965c02521822e92baad09dde15064ac0f326d861533725de283bac6e0ce3618
size 108108
Code/Tools/AssetProcessor/native/ui/style/lyassetprocessor.png
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:68f33fe204a433f8c765d524c9b9e42963f5d16c3442f36df87185bcb4555111
size 8686
Code/Tools/AssetProcessor/native/ui/style/o3de_assetprocessor.ico
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:424c7e9a5bc4819e838e142ae87c987ce96ef40ff853d4a2610d7f068d635851
size 107989
Code/Tools/AssetProcessor/native/ui/style/o3de_assetprocessor.png
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9f16c891aaa1686a3735fe84c7a69c3cef24e68af7baa86bf2f54ab4d51b71e8
size 8489
Code/Tools/AssetProcessor/native/unittests/AssetProcessingStateDataUnitTests.cpp
+13 -13
View File
@@ -148,7 +148,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
scanFolders.clear();
//add a scanfolder
scanFolder = ScanFolderDatabaseEntry("c:/lumberyard/dev", "dev", "rootportkey", "");
scanFolder = ScanFolderDatabaseEntry("c:/O3DE/dev", "dev", "rootportkey", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(scanFolder));
if (scanFolder.m_scanFolderID == AzToolsFramework::AssetDatabase::InvalidEntryId)
{
@@ -158,7 +158,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
//add the same folder again, should not add another because it already exists, so we should get the same id
// not only that, but the path should update.
ScanFolderDatabaseEntry dupeScanFolder = ScanFolderDatabaseEntry("c:/lumberyard/dev2", "dev", "rootportkey", "");
ScanFolderDatabaseEntry dupeScanFolder = ScanFolderDatabaseEntry("c:/O3DE/dev2", "dev", "rootportkey", "");
dupeScanFolder.m_scanFolderID = AzToolsFramework::AssetDatabase::InvalidEntryId;
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(dupeScanFolder));
if (!(dupeScanFolder == scanFolder))
@@ -174,7 +174,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
scanFolders.clear();
UNIT_TEST_EXPECT_TRUE(stateData->GetScanFolders(scanFolders));
UNIT_TEST_EXPECT_TRUE(scanFolders.size() == 1);
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, scanFolder.m_scanFolderID));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainPortableKey(scanFolders, scanFolder.m_portableKey.c_str()));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainPortableKey(scanFolders, "rootportkey"));
@@ -200,7 +200,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
}
//add another folder
ScanFolderDatabaseEntry gameScanFolderEntry("c:/lumberyard/game", "game", "gameportkey", "");
ScanFolderDatabaseEntry gameScanFolderEntry("c:/O3DE/game", "game", "gameportkey", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(gameScanFolderEntry));
if (gameScanFolderEntry.m_scanFolderID == AzToolsFramework::AssetDatabase::InvalidEntryId ||
gameScanFolderEntry.m_scanFolderID == scanFolder.m_scanFolderID)
@@ -213,8 +213,8 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
scanFolders.clear();
UNIT_TEST_EXPECT_TRUE(stateData->GetScanFolders(scanFolders));
UNIT_TEST_EXPECT_TRUE(scanFolders.size() == 2);
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/game"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/game"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, scanFolder.m_scanFolderID));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, gameScanFolderEntry.m_scanFolderID));
@@ -226,11 +226,11 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
scanFolders.clear();
UNIT_TEST_EXPECT_TRUE(stateData->GetScanFolders(scanFolders));
UNIT_TEST_EXPECT_TRUE(scanFolders.size() == 1);
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, scanFolder.m_scanFolderID));
//add another folder again
gameScanFolderEntry = ScanFolderDatabaseEntry("c:/lumberyard/game", "game", "gameportkey2", "");
gameScanFolderEntry = ScanFolderDatabaseEntry("c:/O3DE/game", "game", "gameportkey2", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(gameScanFolderEntry));
if (gameScanFolderEntry.m_scanFolderID == AzToolsFramework::AssetDatabase::InvalidEntryId ||
gameScanFolderEntry.m_scanFolderID == scanFolder.m_scanFolderID)
@@ -243,8 +243,8 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
scanFolders.clear();
UNIT_TEST_EXPECT_TRUE(stateData->GetScanFolders(scanFolders));
UNIT_TEST_EXPECT_TRUE(scanFolders.size() == 2);
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/lumberyard/game"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/dev2"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanPath(scanFolders, "c:/O3DE/game"));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, scanFolder.m_scanFolderID));
UNIT_TEST_EXPECT_TRUE(ScanFoldersContainScanFolderID(scanFolders, gameScanFolderEntry.m_scanFolderID));
@@ -258,7 +258,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
///////////////////////////////////////////////////////////
//setup for sources tests
//for the rest of the test lets add the original scan folder
scanFolder = ScanFolderDatabaseEntry("c:/lumberyard/dev", "dev", "devkey2", "");
scanFolder = ScanFolderDatabaseEntry("c:/O3DE/dev", "dev", "devkey2", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(scanFolder));
///////////////////////////////////////////////////////////
@@ -370,7 +370,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
UNIT_TEST_EXPECT_TRUE(SourcesContainSourceGuid(sources, source.m_sourceGuid));
//add the same source again, but change the scan folder. This should NOT add a new source - even if we don't know what the sourceID is:
ScanFolderDatabaseEntry scanfolder2 = ScanFolderDatabaseEntry("c:/lumberyard/dev2", "dev2", "devkey3", "");
ScanFolderDatabaseEntry scanfolder2 = ScanFolderDatabaseEntry("c:/O3DE/dev2", "dev2", "devkey3", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(scanfolder2));
SourceDatabaseEntry dupeSource2(source);
@@ -554,7 +554,7 @@ void AssetProcessingStateDataUnitTest::DataTest(AssetProcessor::AssetDatabaseCon
////////////////////////////////////////////////////////////////
//Setup for jobs tests by having a scan folder and some sources
//Add a scan folder
scanFolder = ScanFolderDatabaseEntry("c:/lumberyard/dev", "dev", "devkey3", "");
scanFolder = ScanFolderDatabaseEntry("c:/O3DE/dev", "dev", "devkey3", "");
UNIT_TEST_EXPECT_TRUE(stateData->SetScanFolder(scanFolder));
//Add some sources
Code/Tools/AssetProcessor/native/utilities/GUIApplicationManager.cpp
+6 -55
View File
@@ -115,9 +115,6 @@ ApplicationManager::BeforeRunStatus GUIApplicationManager::BeforeRun()
#endif
AssetProcessor::MessageInfoBus::Handler::BusConnect();
QString bootstrapPath = devRoot.filePath("bootstrap.cfg");
m_qtFileWatcher.addPath(bootstrapPath);
// we have to monitor both the cache folder and the database file and restart AP if either of them gets deleted
// It is important to note that we are monitoring the parent folder and not the actual cache folder itself since
// we want to handle the use case on Mac OS if the user moves the cache folder to the trash.
@@ -135,33 +132,6 @@ ApplicationManager::BeforeRunStatus GUIApplicationManager::BeforeRun()
QObject::connect(&m_qtFileWatcher, &QFileSystemWatcher::fileChanged, this, &GUIApplicationManager::FileChanged);
QObject::connect(&m_qtFileWatcher, &QFileSystemWatcher::directoryChanged, this, &GUIApplicationManager::DirectoryChanged);
// Register a notifier for when the project_path property changes within the SettingsRegistry
if (auto settingsRegistry = AZ::SettingsRegistry::Get(); settingsRegistry != nullptr)
{
// Needs to be updated to project_path.
auto OnProjectPathChanged = [this, cachedProjectPath = projectPath](AZStd::string_view path, AZ::SettingsRegistryInterface::Type)
{
constexpr auto projectPathKey =
AZ::SettingsRegistryInterface::FixedValueString(AZ::SettingsRegistryMergeUtils::BootstrapSettingsRootKey)
+ "/project_path";
if (projectPathKey == path)
{
AZ::SettingsRegistryInterface::FixedValueString newProjectPath;
if (auto registry = AZ::SettingsRegistry::Get(); registry && registry->Get(newProjectPath, path))
{
// we only have to quit if the project path has changed, not if just the bootstrap has changed.
if (cachedProjectPath.compare(newProjectPath.c_str()) != 0)
{
AZ_TracePrintf(AssetProcessor::ConsoleChannel, "bootstrap.cfg Project Path changed from %s to %s. Quitting\n",
cachedProjectPath.toUtf8().constData(), newProjectPath.c_str());
QMetaObject::invokeMethod(this, "QuitRequested", Qt::QueuedConnection);
}
}
}
};
m_bootstrapGameFolderChangedHandler = settingsRegistry->RegisterNotifier(AZStd::move(OnProjectPathChanged));
}
return ApplicationManager::BeforeRunStatus::Status_Success;
}
@@ -306,7 +276,7 @@ bool GUIApplicationManager::Run()
m_trayIcon = new QSystemTrayIcon(m_mainWindow);
m_trayIcon->setContextMenu(trayIconMenu);
m_trayIcon->setToolTip(QObject::tr("Asset Processor"));
m_trayIcon->setIcon(QIcon(":/lyassetprocessor.png"));
m_trayIcon->setIcon(QIcon(":/o3de_assetprocessor.png"));
m_trayIcon->show();
QObject::connect(m_trayIcon, &QSystemTrayIcon::activated, m_mainWindow, [&, wrapper](QSystemTrayIcon::ActivationReason reason)
{
@@ -328,8 +298,8 @@ bool GUIApplicationManager::Run()
if (startHidden)
{
m_trayIcon->showMessage(
QCoreApplication::translate("Tray Icon", "Lumberyard Asset Processor has started"),
QCoreApplication::translate("Tray Icon", "The Lumberyard Asset Processor monitors raw project assets and converts those assets into runtime-ready data."),
QCoreApplication::translate("Tray Icon", "Open 3D Engine Asset Processor has started"),
QCoreApplication::translate("Tray Icon", "The Open 3D Engine Asset Processor monitors raw project assets and converts those assets into runtime-ready data."),
QSystemTrayIcon::Information, 3000);
}
}
@@ -651,29 +621,10 @@ void GUIApplicationManager::DirectoryChanged([[maybe_unused]] QString path)
void GUIApplicationManager::FileChanged(QString path)
{
QDir devRoot = ApplicationManager::GetSystemRoot();
QString bootstrapPath = devRoot.filePath("bootstrap.cfg");
QDir projectCacheRoot;
AssetUtilities::ComputeProjectCacheRoot(projectCacheRoot);
QString assetDbPath = projectCacheRoot.filePath("assetdb.sqlite");
if (QString::compare(AssetUtilities::NormalizeFilePath(path), bootstrapPath, Qt::CaseInsensitive) == 0)
{
AssetUtilities::UpdateBranchToken();
if (m_connectionManager)
{
m_connectionManager->UpdateAllowedListFromBootStrap();
}
// Re-merge the Bootstrap.cfg into the SettingsRegistry
AZStd::vector<char> scratchBuffer;
AZ::SettingsRegistryInterface* settingsRegistry = AZ::SettingsRegistry::Get();
AZ_Assert(settingsRegistry, "Unable to retrieve global SettingsRegistry, it should be available now");
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_Bootstrap(*settingsRegistry);
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_O3deUserRegistry(*settingsRegistry, AZ_TRAIT_OS_PLATFORM_CODENAME, {});
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_CommandLine(*settingsRegistry, *m_frameworkApp.GetAzCommandLine(), false);
AZ::SettingsRegistryMergeUtils::MergeSettingsToRegistry_AddRuntimeFilePaths(*settingsRegistry);
}
else if (QString::compare(AssetUtilities::NormalizeFilePath(path), assetDbPath, Qt::CaseInsensitive) == 0)
if (QString::compare(AssetUtilities::NormalizeFilePath(path), assetDbPath, Qt::CaseInsensitive) == 0)
{
if (!QFile::exists(assetDbPath))
{
@@ -868,7 +819,7 @@ void GUIApplicationManager::ShowTrayIconErrorMessage(QString msg)
{
m_timeWhenLastWarningWasShown = currentTime;
m_trayIcon->showMessage(
QCoreApplication::translate("Tray Icon", "Lumberyard Asset Processor"),
QCoreApplication::translate("Tray Icon", "Open 3D Engine Asset Processor"),
QCoreApplication::translate("Tray Icon", msg.toUtf8().data()),
QSystemTrayIcon::Critical, 3000);
}
@@ -880,7 +831,7 @@ void GUIApplicationManager::ShowTrayIconMessage(QString msg)
if (m_trayIcon && m_mainWindow && !m_mainWindow->isVisible())
{
m_trayIcon->showMessage(
QCoreApplication::translate("Tray Icon", "Lumberyard Asset Processor"),
QCoreApplication::translate("Tray Icon", "Open 3D Engine Asset Processor"),
QCoreApplication::translate("Tray Icon", msg.toUtf8().data()),
QSystemTrayIcon::Information, 3000);
}
Code/Tools/AssetProcessor/native/utilities/GUIApplicationManager.h
-1
View File
@@ -112,7 +112,6 @@ private:
QPointer<QSystemTrayIcon> m_trayIcon;
QPointer<MainWindow> m_mainWindow;
AZ::SettingsRegistryInterface::NotifyEventHandler m_bootstrapGameFolderChangedHandler;
AZStd::chrono::system_clock::time_point m_timeWhenLastWarningWasShown;
};
Code/Tools/AssetProcessor/native/utilities/assetUtils.cpp
+18 -2
View File
@@ -181,13 +181,29 @@ namespace AssetUtilsInternal
if (AZ::SettingsRegistryMergeUtils::DumpSettingsRegistryToStream(settingsRegistry, AssetProcessorUserSettingsRootKey,
apSettingsStream, apDumperSettings))
{
constexpr const char* AssetProcessorTmpSetreg = "asset_processor.setreg.tmp";
// Write to a temporary file first before renaming it to the final file location
// This is needed to reduce the potential of a race condition which occurs when other applications attempt to load settings registry
// files from the project's user Registry folder while the AssetProcessor is writing the file out the asset_processor.setreg
// at the same time
QString tempDirValue;
AssetUtilities::CreateTempWorkspace(tempDirValue);
QDir tempDir(tempDirValue);
AZ::IO::FixedMaxPath tmpSetregPath = tempDir.absoluteFilePath(QString(AssetProcessorTmpSetreg)).toUtf8().data();
constexpr auto modeFlags = AZ::IO::SystemFile::SF_OPEN_WRITE_ONLY | AZ::IO::SystemFile::SF_OPEN_CREATE
| AZ::IO::SystemFile::SF_OPEN_CREATE_PATH;
if (AZ::IO::SystemFile apSetregFile; apSetregFile.Open(setregPath.c_str(), modeFlags))
if (AZ::IO::SystemFile apSetregFile; apSetregFile.Open(tmpSetregPath.c_str(), modeFlags))
{
size_t bytesWritten = apSetregFile.Write(apSettingsJson.data(), apSettingsJson.size());
return bytesWritten == apSettingsJson.size();
// Close the file so that it can be renamed.
apSetregFile.Close();
if (bytesWritten == apSettingsJson.size())
{
// Create the directory to contain the moved setreg file
AZ::IO::SystemFile::CreateDir(AZ::IO::FixedMaxPath(setregPath.ParentPath()).c_str());
return AZ::IO::SystemFile::Rename(tmpSetregPath.c_str(), setregPath.c_str(), true);
}
}
else
{