Initial commit

Code/Tools/DeltaCataloger/CMakeLists.txt

@@ -0,0 +1,47 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+if(NOT PAL_TRAIT_BUILD_HOST_TOOLS)
+    return()
+endif()
+
+ly_add_target(
+    NAME DeltaCataloger EXECUTABLE
+    NAMESPACE AZ
+    FILES_CMAKE
+        deltacataloger_files.cmake
+    BUILD_DEPENDENCIES
+        PRIVATE
+            AZ::AzCore
+            AZ::AzFramework
+            AZ::AzToolsFramework
+)
+
+if(PAL_TRAIT_BUILD_TESTS_SUPPORTED)
+
+    ly_add_target(
+        NAME DeltaCataloger.Tests ${PAL_TRAIT_TEST_TARGET_TYPE}
+        NAMESPACE AZ
+        FILES_CMAKE
+            deltacataloger_test_files.cmake
+        BUILD_DEPENDENCIES
+            PRIVATE
+                AZ::AzCore
+                AZ::AzFramework
+                AZ::AzTest
+                AZ::AzToolsFramework
+    )
+
+    ly_add_googletest(
+        NAME AZ::DeltaCataloger.Tests
+    )
+
+endif()

Code/Tools/DeltaCataloger/Tests/tests_main.cpp

@@ -0,0 +1,171 @@
+/*
+ * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ * its licensors.
+ *
+ * For complete copyright and license terms please see the LICENSE at the root of this
+ * distribution (the "License"). All use of this software is governed by the License,
+ * or, if provided, by the license below or the license accompanying this file. Do not
+ * remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ */
+
+#include <AzTest/AzTest.h>
+#include <AzCore/Memory/OSAllocator.h>
+#include <AzCore/Memory/SystemAllocator.h>
+#include <AzCore/std/containers/vector.h>
+#include <AzCore/std/string/string.h>
+#include <AzFramework/Asset/AssetBundleManifest.h>
+#include <AzFramework/StringFunc/StringFunc.h>
+#include <AzToolsFramework/Application/ToolsApplication.h>
+#include <AzToolsFramework/AssetBundle/AssetBundleAPI.h>
+#include <AzToolsFramework/AssetBundle/AssetBundleComponent.h>
+#include <AzCore/UnitTest/TestTypes.h>
+#include <AzCore/Debug/TraceMessageBus.h>
+#include <AzCore/UserSettings/UserSettingsComponent.h>
+
+class AssetBundleComponentTests
+    : public UnitTest::ScopedAllocatorSetupFixture,
+    public AZ::Debug::TraceMessageBus::Handler
+{
+public:
+    const char* sourcePakPath = "dir1/dir2/some_test_pak.pak";
+    AZStd::vector<AZStd::string> fileEntriesHasCatalog;
+    AZStd::vector<AZStd::string> fileEntriesNoCatalog;
+    AZStd::string catalogPath;
+
+    AzToolsFramework::ToolsApplication app;
+
+    using AssetBundleCommandsBus = AzToolsFramework::AssetBundleCommandsBus;
+
+    AZStd::string CreateCatalogPrefix() const
+    {
+        return AzToolsFramework::AssetBundleComponent::DeltaCatalogName;
+    }
+protected:
+    void SetUp() override
+    {
+        AZ::ComponentApplication::Descriptor desc;
+        desc.m_useExistingAllocator = true;
+        desc.m_enableDrilling = false; // we already created a memory driller for the test (AllocatorsFixture)
+        app.Start(desc);
+
+        // Without this, the user settings component would attempt to save on finalize/shutdown. Since the file is
+        // shared across the whole engine, if multiple tests are run in parallel, the saving could cause a crash 
+        // in the unit tests.
+        AZ::UserSettingsComponentRequestBus::Broadcast(&AZ::UserSettingsComponentRequests::DisableSaveOnFinalize);
+
+        catalogPath = AZStd::string::format("%s.111111.xml", CreateCatalogPrefix().c_str());
+        
+        // normalize paths before inserting them in the containers
+        AZStd::string sourcePak(sourcePakPath);
+        AzFramework::StringFunc::Path::Normalize(sourcePak);
+        fileEntriesHasCatalog.push_back(sourcePak);
+        
+        AzFramework::StringFunc::Path::Normalize(catalogPath);
+        fileEntriesHasCatalog.push_back(catalogPath);
+        
+        fileEntriesHasCatalog.push_back(AzFramework::AssetBundleManifest::s_manifestFileName);
+        
+        AZStd::string firstDummyPath("basePath/somePath1");
+        AzFramework::StringFunc::Path::Normalize(firstDummyPath);
+        fileEntriesHasCatalog.emplace_back(firstDummyPath);
+        
+        AZStd::string secondDummyPath("somePath2");
+        AzFramework::StringFunc::Path::Normalize(secondDummyPath);
+        fileEntriesHasCatalog.emplace_back(secondDummyPath);
+
+        fileEntriesNoCatalog.push_back(sourcePak);
+        fileEntriesNoCatalog.emplace_back(firstDummyPath);
+        fileEntriesNoCatalog.emplace_back(secondDummyPath);
+    }
+
+
+    bool OnPreError([[maybe_unused]] const char* window, [[maybe_unused]] const char* fileName, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* message)
+    {
+        return true;
+    }
+
+    void TearDown() override
+    {
+        app.Stop();
+    }
+};
+
+TEST_F(AssetBundleComponentTests, HasManifest_ManifestInBundle_ExpectTrue)
+{
+    AZStd::vector<AZStd::string> fileEntries;
+    fileEntries.push_back(AzFramework::AssetBundleManifest::s_manifestFileName);
+    EXPECT_TRUE(AzToolsFramework::AssetBundleComponent::HasManifest(fileEntries));
+}
+TEST_F(AssetBundleComponentTests, HasManifest_ManifestNotInBundle_ExpectFalse)
+{
+    AZStd::vector<AZStd::string> fileEntries;
+    fileEntries.push_back("randomString");
+    EXPECT_FALSE(AzToolsFramework::AssetBundleComponent::HasManifest(fileEntries));
+}
+
+TEST_F(AssetBundleComponentTests, RemoveNonAssetEntries_HasManifest_NotFound)
+{
+    AZStd::string normalizedSourcePakPath = sourcePakPath;
+    AzFramework::StringFunc::Path::Normalize(normalizedSourcePakPath);
+    
+    AzFramework::AssetBundleManifest manifest;
+    manifest.SetCatalogName(AZStd::string::format("%s.111111.xml", CreateCatalogPrefix().c_str()));
+    
+    bool result = AzToolsFramework::AssetBundleComponent::RemoveNonAssetFileEntries(fileEntriesHasCatalog, normalizedSourcePakPath, &manifest);
+    EXPECT_TRUE(result);
+
+    // check to make sure that sourcePakPath doesn't exist in fileEntriesHasCatalog
+    auto itr = AZStd::find(fileEntriesHasCatalog.begin(), fileEntriesHasCatalog.end(), normalizedSourcePakPath);
+    EXPECT_EQ(itr, fileEntriesHasCatalog.end());
+
+    // check to make sure that manifest doesn't exist in fileEntriesHasCatalog
+    itr = AZStd::find(fileEntriesHasCatalog.begin(), fileEntriesHasCatalog.end(), AZStd::string(AzFramework::AssetBundleManifest::s_manifestFileName));
+    EXPECT_EQ(itr, fileEntriesHasCatalog.end());
+
+    // check to make sure that the catalog doesn't exist in fileEntriesHasCatalog
+    itr = AZStd::find(fileEntriesHasCatalog.begin(), fileEntriesHasCatalog.end(), manifest.GetCatalogName());
+    EXPECT_EQ(itr, fileEntriesHasCatalog.end());
+}
+
+TEST_F(AssetBundleComponentTests, RemoveNonAssetEntries_HasManifestCatalog_FailedToFindCatalog)
+{
+    AZStd::string normalizedSourcePakPath = sourcePakPath;
+    AzFramework::StringFunc::Path::Normalize(normalizedSourcePakPath);
+
+    AzFramework::AssetBundleManifest manifest;
+    manifest.SetCatalogName(AZStd::string::format("%s.22222.xml", CreateCatalogPrefix().c_str()));
+
+    AZ::Debug::TraceMessageBus::Handler::BusConnect();
+    bool result = AzToolsFramework::AssetBundleComponent::RemoveNonAssetFileEntries(fileEntriesHasCatalog, normalizedSourcePakPath, &manifest);
+    EXPECT_FALSE(result);
+    AZ::Debug::TraceMessageBus::Handler::BusDisconnect();
+
+    // check to make sure that sourcePakPath doesn't exist in fileEntriesHasCatalog
+    auto itr = AZStd::find(fileEntriesHasCatalog.begin(), fileEntriesHasCatalog.end(), normalizedSourcePakPath);
+    EXPECT_EQ(itr, fileEntriesHasCatalog.end());
+
+    // check to make sure that manifest doesn't exist in fileEntriesHasCatalog
+    itr = AZStd::find(fileEntriesHasCatalog.begin(), fileEntriesHasCatalog.end(), AZStd::string(AzFramework::AssetBundleManifest::s_manifestFileName));
+    EXPECT_EQ(itr, fileEntriesHasCatalog.end());
+
+    // check to make sure that the catalog doesn't exist in 
+    itr = AZStd::find(fileEntriesHasCatalog.begin(), fileEntriesHasCatalog.end(), manifest.GetCatalogName());
+    EXPECT_EQ(itr, fileEntriesHasCatalog.end());
+}
+
+TEST_F(AssetBundleComponentTests, RemoveNonAssetEntries_PakAssetEntryWasRemoved_Success)
+{
+    AZStd::string normalizedSourcePakPath = sourcePakPath;
+    AzFramework::StringFunc::Path::Normalize(normalizedSourcePakPath);
+
+    bool result = AzToolsFramework::AssetBundleComponent::RemoveNonAssetFileEntries(fileEntriesHasCatalog, normalizedSourcePakPath, nullptr);
+    EXPECT_TRUE(result);
+
+    // check to make sure that sourcePakPath doesn't exist in fileEntriesHasCatalog
+    auto itr = AZStd::find(fileEntriesHasCatalog.begin(), fileEntriesHasCatalog.end(), normalizedSourcePakPath);
+    EXPECT_EQ(itr, fileEntriesHasCatalog.end());
+}
+
+AZ_UNIT_TEST_HOOK(DEFAULT_UNIT_TEST_ENV);

Code/Tools/DeltaCataloger/deltacataloger_files.cmake

@@ -0,0 +1,14 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+set(FILES
+    source/main.cpp
+)

Code/Tools/DeltaCataloger/deltacataloger_test_files.cmake

@@ -0,0 +1,14 @@
+#
+# All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+# its licensors.
+#
+# For complete copyright and license terms please see the LICENSE at the root of this
+# distribution (the "License"). All use of this software is governed by the License,
+# or, if provided, by the license below or the license accompanying this file. Do not
+# remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+
+set(FILES
+    Tests/tests_main.cpp
+)

Code/Tools/DeltaCataloger/source/main.cpp

@@ -0,0 +1,167 @@
+/*
+ * All or portions of this file Copyright (c) Amazon.com, Inc. or its affiliates or
+ * its licensors.
+ *
+ * For complete copyright and license terms please see the LICENSE at the root of this
+ * distribution (the "License"). All use of this software is governed by the License,
+ * or, if provided, by the license below or the license accompanying this file. Do not
+ * remove or modify any license notices. This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ */
+
+#include <AzCore/Memory/AllocatorManager.h>
+#include <AzToolsFramework/Application/ToolsApplication.h>
+
+#include <AzCore/Asset/AssetManagerBus.h>
+#include <AzCore/IO/FileIO.h>
+#include <AzCore/Serialization/ObjectStream.h>
+#include <AzCore/Serialization/Utils.h>
+#include <AzFramework/Asset/AssetBundleManifest.h>
+#include <AzFramework/CommandLine/CommandLine.h>
+#include <AzFramework/StringFunc/StringFunc.h>
+#include <AzToolsFramework/Archive/ArchiveAPI.h>
+#include <AzToolsFramework/AssetBundle/AssetBundleAPI.h>
+
+const char* appWindowName = "DeltaCataloger";
+enum class DeltaCatalogerResult : AZ::u8
+{
+    Success = 0,
+
+    InvalidArg = 1,
+    FailedToCreateDeltaCatalog,
+    FailedToInjectFile, 
+};
+
+struct DeltaCatalogerParams
+{
+    AZStd::string sourceCatalogPath;
+    AZStd::vector<AZStd::string> sourcePaks;
+    AZStd::string workingDirectory;
+    bool verbose = false;
+    bool regenerateExistingDeltas = false;
+};
+
+
+DeltaCatalogerResult DeltaCataloger(DeltaCatalogerParams& params)
+{
+    using AssetCatalogRequestBus = AZ::Data::AssetCatalogRequestBus;
+    using AssetBundleCommandsBus = AzToolsFramework::AssetBundleCommands::Bus;
+
+    // update all relative paths given to be relative to the working directory
+    if (params.workingDirectory.length())
+    { 
+        AzFramework::StringFunc::Path::Join(params.workingDirectory.c_str(), params.sourceCatalogPath.c_str(), params.sourceCatalogPath);
+        for (AZ::u32 index = 0; index < params.sourcePaks.size(); ++index)
+        {
+            AzFramework::StringFunc::Path::Join(params.workingDirectory.c_str(), params.sourcePaks[index].c_str(), params.sourcePaks[index]);
+        }
+    }
+
+    // validate params
+    AZ::IO::FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
+    if (!fileIO->Exists(params.sourceCatalogPath.c_str()))
+    {
+        AZ_Error(appWindowName, false, "Invalid Arg: Source Asset Catalog does not exist at \"%s\".", params.sourceCatalogPath.c_str());
+        return DeltaCatalogerResult::InvalidArg;
+    }
+
+    if (params.sourcePaks.empty())
+    {
+        AZ_Error(appWindowName, false, "Failed to read source pak files arg list. Should start from second argument.");
+        return DeltaCatalogerResult::InvalidArg;
+    }
+
+    for (const AZStd::string& sourcePakPath : params.sourcePaks)
+    {
+        if (!fileIO->Exists(sourcePakPath.c_str()))
+        {
+            AZ_Error(appWindowName, false, "Invalid Arg: Source Pak does not exist at \"%s\".", sourcePakPath.c_str());
+            return DeltaCatalogerResult::InvalidArg;
+        }
+    }
+  
+    // Load the source catalog
+    if (params.verbose)
+    {
+        AZ_TracePrintf(appWindowName, "Loading source asset catalog \"%s\".\n", params.sourceCatalogPath.c_str())
+    }
+
+    AssetCatalogRequestBus::Broadcast(&AssetCatalogRequestBus::Events::ClearCatalog);
+    bool result = false;
+    AssetCatalogRequestBus::BroadcastResult(result, &AssetCatalogRequestBus::Events::LoadCatalog, params.sourceCatalogPath.c_str());
+
+    if (result)
+    {
+        for (const AZStd::string& sourcePakPath : params.sourcePaks)
+        {
+            bool catalogCreated = false;
+            AssetBundleCommandsBus::BroadcastResult(catalogCreated, &AssetBundleCommandsBus::Events::CreateDeltaCatalog, sourcePakPath, params.regenerateExistingDeltas);
+            if (!catalogCreated)
+            {
+                AZ_Error(appWindowName, false, "Failed to make or inject delta asset catalog for \"%s\".", sourcePakPath.c_str());
+                return DeltaCatalogerResult::FailedToCreateDeltaCatalog;
+            }
+        }
+    }
+    else
+    {
+        AZStd::string error = AZStd::string::format("Failed to load source asset catalog \"%s\".", params.sourceCatalogPath.c_str());
+        AZ_Error(appWindowName, false, error.c_str());
+        return DeltaCatalogerResult::FailedToCreateDeltaCatalog;
+    }
+    return DeltaCatalogerResult::Success;
+}
+
+
+DeltaCatalogerResult ParseArgs(const AzFramework::CommandLine* parser, DeltaCatalogerParams& params)
+{
+    // AzFramework CommandLine consumes the first arg (the executable itself), so positional or switch args start at 0
+    const AZ::u8 sourceCatalogPathIndex = 0;
+    const AZ::u8 sourceCatalogStartIndex = 1;
+
+    params.sourceCatalogPath = parser->GetMiscValue(sourceCatalogPathIndex);
+    AZ::u64 numPositionalArgs = parser->GetNumMiscValues();
+    for (AZ::u64 index = sourceCatalogStartIndex; index < numPositionalArgs; ++index)
+    {
+        params.sourcePaks.push_back(parser->GetMiscValue(index));
+    }
+
+    params.verbose = parser->HasSwitch("verbose");
+    params.regenerateExistingDeltas = parser->HasSwitch("regenerate");
+    params.workingDirectory = parser->GetSwitchValue("working-dir", 0);
+    
+    return DeltaCatalogerResult::Success;
+}
+
+int main(int argc, char** argv)
+{
+    DeltaCatalogerResult exitCode = DeltaCatalogerResult::Success;
+    const AZ::u8 minimumArgCount = 3; // 0 = exe, 1 = source catalog path, 2 = source pak path, from raw commandline input
+    if (argc < minimumArgCount)
+    {
+        AZ_Error(appWindowName, false, "Must specify source catalog, and at least one source pak file.");
+        return static_cast<int>(DeltaCatalogerResult::InvalidArg);
+    }
+
+    AzToolsFramework::ToolsApplication app(&argc, &argv);
+
+    // can deal with starting the app with a different working directory if needed later (use lmbr cli main.cpp as a reference)
+    AzFramework::Application::StartupParameters startParam;
+    app.Start(AzFramework::Application::Descriptor());    
+    {
+        DeltaCatalogerParams params;
+        exitCode = ParseArgs(app.GetCommandLine(), params);
+        if (exitCode != DeltaCatalogerResult::Success)
+        {
+            return static_cast<int>(exitCode);
+        }
+
+        exitCode = DeltaCataloger(params);
+
+        // Tick until everything is ready for shutdown
+        app.Tick();
+    }
+    app.Stop();    
+    return static_cast<int>(exitCode);
+}