diff --git a/Assets/Engine/SeedAssetList.seed b/Assets/Engine/SeedAssetList.seed index 77ec509721..579fd3c444 100644 --- a/Assets/Engine/SeedAssetList.seed +++ b/Assets/Engine/SeedAssetList.seed @@ -66,779 +66,75 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1384,166 +680,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1632,46 +768,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/AutomatedTesting/AutomatedTesting_Dependencies.xml b/AutomatedTesting/AutomatedTesting_Dependencies.xml index 50a5caea73..98e00a2914 100644 --- a/AutomatedTesting/AutomatedTesting_Dependencies.xml +++ b/AutomatedTesting/AutomatedTesting_Dependencies.xml @@ -1,5 +1,4 @@ - \ No newline at end of file diff --git a/Code/Framework/AzCore/AzCore/IO/Streamer/Scheduler.cpp b/Code/Framework/AzCore/AzCore/IO/Streamer/Scheduler.cpp index 9b4996ad49..9ee0fefc99 100644 --- a/Code/Framework/AzCore/AzCore/IO/Streamer/Scheduler.cpp +++ b/Code/Framework/AzCore/AzCore/IO/Streamer/Scheduler.cpp @@ -43,10 +43,12 @@ namespace AZ::IO m_mainLoopDesc = threadDesc; m_mainLoopDesc.m_name = "IO Scheduler"; - m_mainLoop = AZStd::thread([this]() - { - Thread_MainLoop(); - }, &m_mainLoopDesc); + m_mainLoop = AZStd::thread( + m_mainLoopDesc, + [this]() + { + Thread_MainLoop(); + }); } } diff --git a/Code/Framework/AzCore/AzCore/Jobs/Internal/JobManagerWorkStealing.cpp b/Code/Framework/AzCore/AzCore/Jobs/Internal/JobManagerWorkStealing.cpp index f76946a667..230bf959f6 100644 --- a/Code/Framework/AzCore/AzCore/Jobs/Internal/JobManagerWorkStealing.cpp +++ b/Code/Framework/AzCore/AzCore/Jobs/Internal/JobManagerWorkStealing.cpp @@ -644,11 +644,11 @@ JobManagerWorkStealing::ThreadList JobManagerWorkStealing::CreateWorkerThreads(c } info->m_thread = AZStd::thread( + threadDesc, [this, info]() { this->ProcessJobsWorker(info); - }, - &threadDesc + } ); info->m_threadId = info->m_thread.get_id(); diff --git a/Code/Framework/AzCore/AzCore/Task/TaskExecutor.cpp b/Code/Framework/AzCore/AzCore/Task/TaskExecutor.cpp index d2d19d1359..7da04d7301 100644 --- a/Code/Framework/AzCore/AzCore/Task/TaskExecutor.cpp +++ b/Code/Framework/AzCore/AzCore/Task/TaskExecutor.cpp @@ -205,13 +205,13 @@ namespace AZ } m_active.store(true, AZStd::memory_order_release); - m_thread = AZStd::thread{ [this, &initSemaphore] + m_thread = AZStd::thread{ desc, + [this, &initSemaphore] { t_worker = this; initSemaphore.release(); Run(); - }, - &desc }; + } }; } // Threads that wait on a graph to complete are disqualified from receiving tasks until the wait finishes diff --git a/Code/Framework/AzCore/AzCore/std/parallel/thread.h b/Code/Framework/AzCore/AzCore/std/parallel/thread.h index 15d8c9dc8e..eef269c8ac 100644 --- a/Code/Framework/AzCore/AzCore/std/parallel/thread.h +++ b/Code/Framework/AzCore/AzCore/std/parallel/thread.h @@ -87,12 +87,6 @@ namespace AZStd // construct/copy/destroy: thread(); - /** - * \note thread_desc is AZStd extension. - */ - template - explicit thread(F&& f, const thread_desc* desc = 0); - ~thread(); thread(thread&& rhs) @@ -108,6 +102,15 @@ namespace AZStd return *this; } + template, thread_desc>>> + explicit thread(F&& f, Args&&... args); + + /** + * \note thread_desc is AZStd extension. + */ + template + thread(const thread_desc& desc, F&& f, Args&&... args); + // Till we fully have RVALUES template explicit thread(Internal::thread_move_t f); @@ -138,8 +141,8 @@ namespace AZStd //thread(AZStd::delegate d,const thread_desc* desc = 0); private: - thread(thread&); - thread& operator=(thread&); + thread(const thread&) = delete; + thread& operator=(const thread&) = delete; native_thread_data_type m_thread; }; diff --git a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/std/parallel/internal/thread_UnixLike.h b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/std/parallel/internal/thread_UnixLike.h index 499caebac0..d9a4982a0a 100644 --- a/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/std/parallel/internal/thread_UnixLike.h +++ b/Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/std/parallel/internal/thread_UnixLike.h @@ -10,6 +10,8 @@ #include #include +#include + namespace AZStd { namespace Internal @@ -22,12 +24,20 @@ namespace AZStd ////////////////////////////////////////////////////////////////////////// // thread - template - inline thread::thread(F&& f, const thread_desc* desc) + template + thread::thread(F&& f, Args&&... args) + : thread(thread_desc{}, AZStd::forward(f), AZStd::forward(args)...) + {} + + template + thread::thread(const thread_desc& desc, F&& f, Args&&... args) { - Internal::thread_info* ti = Internal::create_thread_info(AZStd::forward(f)); - ti->m_name = desc ? desc->m_name : nullptr; - m_thread = Internal::create_thread(desc, ti); + auto threadfunc = [fn = AZStd::forward(f), argsTuple = AZStd::make_tuple(AZStd::forward(args)...)]() mutable -> void + { + AZStd::apply(AZStd::move(fn), AZStd::move(argsTuple)); + }; + Internal::thread_info* ti = Internal::create_thread_info(AZStd::move(threadfunc)); + m_thread = Internal::create_thread(&desc, ti); } inline bool thread::joinable() const diff --git a/Code/Framework/AzCore/Platform/Common/WinAPI/AzCore/std/parallel/internal/thread_WinAPI.h b/Code/Framework/AzCore/Platform/Common/WinAPI/AzCore/std/parallel/internal/thread_WinAPI.h index 46986521e7..c79381a74a 100644 --- a/Code/Framework/AzCore/Platform/Common/WinAPI/AzCore/std/parallel/internal/thread_WinAPI.h +++ b/Code/Framework/AzCore/Platform/Common/WinAPI/AzCore/std/parallel/internal/thread_WinAPI.h @@ -18,6 +18,8 @@ extern "C" AZ_DLL_IMPORT unsigned long __stdcall GetCurrentThreadId(void); } +#include + namespace AZStd { namespace Internal @@ -30,11 +32,20 @@ namespace AZStd ////////////////////////////////////////////////////////////////////////// // thread - template - inline thread::thread(F&& f, const thread_desc* desc) + template + thread::thread(F&& f, Args&&... args) + : thread(thread_desc{}, AZStd::forward(f), AZStd::forward(args)...) + {} + + template + thread::thread(const thread_desc& desc, F&& f, Args&&... args) { - Internal::thread_info* ti = Internal::create_thread_info(AZStd::forward(f)); - m_thread.m_handle = Internal::create_thread(desc, ti, &m_thread.m_id); + auto threadfunc = [fn = AZStd::forward(f), argsTuple = AZStd::make_tuple(AZStd::forward(args)...)]() mutable -> void + { + AZStd::apply(AZStd::move(fn), AZStd::move(argsTuple)); + }; + Internal::thread_info* ti = Internal::create_thread_info(AZStd::move(threadfunc)); + m_thread.m_handle = Internal::create_thread(&desc, ti, &m_thread.m_id); } inline bool thread::joinable() const diff --git a/Code/Framework/AzCore/Tests/AZStd/Parallel.cpp b/Code/Framework/AzCore/Tests/AZStd/Parallel.cpp index f3d4f58250..407cd3c258 100644 --- a/Code/Framework/AzCore/Tests/AZStd/Parallel.cpp +++ b/Code/Framework/AzCore/Tests/AZStd/Parallel.cpp @@ -195,18 +195,18 @@ namespace UnitTest void test_thread_id_for_running_thread_is_not_default_constructed_id() { - const thread_desc* desc = m_numThreadDesc ? &m_desc[0] : nullptr; - AZStd::thread t(AZStd::bind(&Parallel_Thread::do_nothing, this), desc); + const thread_desc desc = m_numThreadDesc ? m_desc[0] : thread_desc{}; + AZStd::thread t(desc, AZStd::bind(&Parallel_Thread::do_nothing, this)); AZ_TEST_ASSERT(t.get_id() != AZStd::thread::id()); t.join(); } void test_different_threads_have_different_ids() { - const thread_desc* desc1 = m_numThreadDesc ? &m_desc[0] : nullptr; - const thread_desc* desc2 = m_numThreadDesc ? &m_desc[1] : nullptr; - AZStd::thread t(AZStd::bind(&Parallel_Thread::do_nothing, this), desc1); - AZStd::thread t2(AZStd::bind(&Parallel_Thread::do_nothing, this), desc2); + const thread_desc desc1 = m_numThreadDesc ? m_desc[0] : thread_desc{}; + const thread_desc desc2 = m_numThreadDesc ? m_desc[1] : thread_desc{}; + AZStd::thread t(desc1, AZStd::bind(&Parallel_Thread::do_nothing, this)); + AZStd::thread t2(desc2, AZStd::bind(&Parallel_Thread::do_nothing, this)); AZ_TEST_ASSERT(t.get_id() != t2.get_id()); t.join(); t2.join(); @@ -214,13 +214,13 @@ namespace UnitTest void test_thread_ids_have_a_total_order() { - const thread_desc* desc1 = m_numThreadDesc ? &m_desc[0] : nullptr; - const thread_desc* desc2 = m_numThreadDesc ? &m_desc[1] : nullptr; - const thread_desc* desc3 = m_numThreadDesc ? &m_desc[2] : nullptr; + const thread_desc desc1 = m_numThreadDesc ? m_desc[0] : thread_desc{}; + const thread_desc desc2 = m_numThreadDesc ? m_desc[1] : thread_desc{}; + const thread_desc desc3 = m_numThreadDesc ? m_desc[2] : thread_desc{}; - AZStd::thread t(AZStd::bind(&Parallel_Thread::do_nothing, this), desc1); - AZStd::thread t2(AZStd::bind(&Parallel_Thread::do_nothing, this), desc2); - AZStd::thread t3(AZStd::bind(&Parallel_Thread::do_nothing, this), desc3); + AZStd::thread t(desc1, AZStd::bind(&Parallel_Thread::do_nothing, this)); + AZStd::thread t2(desc2, AZStd::bind(&Parallel_Thread::do_nothing, this)); + AZStd::thread t3(desc3, AZStd::bind(&Parallel_Thread::do_nothing, this)); AZ_TEST_ASSERT(t.get_id() != t2.get_id()); AZ_TEST_ASSERT(t.get_id() != t3.get_id()); AZ_TEST_ASSERT(t2.get_id() != t3.get_id()); @@ -313,10 +313,10 @@ namespace UnitTest void test_thread_id_of_running_thread_returned_by_this_thread_get_id() { - const thread_desc* desc1 = m_numThreadDesc ? &m_desc[0] : nullptr; + const thread_desc desc1 = m_numThreadDesc ? m_desc[0] : thread_desc{}; AZStd::thread::id id; - AZStd::thread t(AZStd::bind(&Parallel_Thread::get_thread_id, this, &id), desc1); + AZStd::thread t(desc1, AZStd::bind(&Parallel_Thread::get_thread_id, this, &id)); AZStd::thread::id t_id = t.get_id(); t.join(); AZ_TEST_ASSERT(id == t_id); @@ -366,10 +366,10 @@ namespace UnitTest void test_move_on_construction() { - const thread_desc* desc1 = m_numThreadDesc ? &m_desc[0] : nullptr; + const thread_desc desc1 = m_numThreadDesc ? m_desc[0] : thread_desc{}; AZStd::thread::id the_id; AZStd::thread x; - x = AZStd::thread(AZStd::bind(&Parallel_Thread::do_nothing_id, this, &the_id), desc1); + x = AZStd::thread(desc1, AZStd::bind(&Parallel_Thread::do_nothing_id, this, &the_id)); AZStd::thread::id x_id = x.get_id(); x.join(); AZ_TEST_ASSERT(the_id == x_id); @@ -377,8 +377,8 @@ namespace UnitTest AZStd::thread make_thread(AZStd::thread::id* the_id) { - const thread_desc* desc1 = m_numThreadDesc ? &m_desc[0] : nullptr; - return AZStd::thread(AZStd::bind(&Parallel_Thread::do_nothing_id, this, the_id), desc1); + const thread_desc desc1 = m_numThreadDesc ? m_desc[0] : thread_desc{}; + return AZStd::thread(desc1, AZStd::bind(&Parallel_Thread::do_nothing_id, this, the_id)); } void test_move_from_function_return() @@ -430,9 +430,9 @@ namespace UnitTest void do_test_creation() { - const thread_desc* desc1 = m_numThreadDesc ? &m_desc[0] : nullptr; + const thread_desc desc1 = m_numThreadDesc ? m_desc[0] : thread_desc{}; m_data = 0; - AZStd::thread t(AZStd::bind(&Parallel_Thread::simple_thread, this), desc1); + AZStd::thread t(desc1, AZStd::bind(&Parallel_Thread::simple_thread, this)); t.join(); AZ_TEST_ASSERT(m_data == 999); } @@ -445,9 +445,9 @@ namespace UnitTest void do_test_id_comparison() { - const thread_desc* desc1 = m_numThreadDesc ? &m_desc[0] : nullptr; + const thread_desc desc1 = m_numThreadDesc ? m_desc[0] : thread_desc{}; AZStd::thread::id self = this_thread::get_id(); - AZStd::thread thrd(AZStd::bind(&Parallel_Thread::comparison_thread, this, self), desc1); + AZStd::thread thrd(desc1, AZStd::bind(&Parallel_Thread::comparison_thread, this, self)); thrd.join(); } @@ -476,10 +476,10 @@ namespace UnitTest void do_test_creation_through_reference_wrapper() { - const thread_desc* desc1 = m_numThreadDesc ? &m_desc[0] : nullptr; + const thread_desc desc1 = m_numThreadDesc ? m_desc[0] : thread_desc{}; non_copyable_functor f; - AZStd::thread thrd(AZStd::ref(f), desc1); + AZStd::thread thrd(desc1, AZStd::ref(f)); thrd.join(); AZ_TEST_ASSERT(f.value == 999); } @@ -491,10 +491,10 @@ namespace UnitTest void test_swap() { - const thread_desc* desc1 = m_numThreadDesc ? &m_desc[0] : nullptr; - const thread_desc* desc2 = m_numThreadDesc ? &m_desc[1] : nullptr; - AZStd::thread t(AZStd::bind(&Parallel_Thread::simple_thread, this), desc1); - AZStd::thread t2(AZStd::bind(&Parallel_Thread::simple_thread, this), desc2); + const thread_desc desc1 = m_numThreadDesc ? m_desc[0] : thread_desc{}; + const thread_desc desc2 = m_numThreadDesc ? m_desc[1] : thread_desc{}; + AZStd::thread t(desc1, AZStd::bind(&Parallel_Thread::simple_thread, this)); + AZStd::thread t2(desc2, AZStd::bind(&Parallel_Thread::simple_thread, this)); AZStd::thread::id id1 = t.get_id(); AZStd::thread::id id2 = t2.get_id(); @@ -512,7 +512,7 @@ namespace UnitTest void run() { - const thread_desc* desc1 = m_numThreadDesc ? &m_desc[0] : nullptr; + const thread_desc desc1 = m_numThreadDesc ? m_desc[0] : thread_desc{}; // We need to have at least one processor AZ_TEST_ASSERT(AZStd::thread::hardware_concurrency() >= 1); @@ -520,18 +520,18 @@ namespace UnitTest // Create thread to increment data till we need to m_data = 0; m_dataMax = 10; - AZStd::thread tr(AZStd::bind(&Parallel_Thread::increment_data, this), desc1); + AZStd::thread tr(desc1, AZStd::bind(&Parallel_Thread::increment_data, this)); tr.join(); AZ_TEST_ASSERT(m_data == m_dataMax); m_data = 0; - AZStd::thread trDel(make_delegate(this, &Parallel_Thread::increment_data), desc1); + AZStd::thread trDel(desc1, make_delegate(this, &Parallel_Thread::increment_data)); trDel.join(); AZ_TEST_ASSERT(m_data == m_dataMax); chrono::system_clock::time_point startTime = chrono::system_clock::now(); { - AZStd::thread tr1(AZStd::bind(&Parallel_Thread::sleep_thread, this, chrono::milliseconds(100)), desc1); + AZStd::thread tr1(desc1, AZStd::bind(&Parallel_Thread::sleep_thread, this, chrono::milliseconds(100))); tr1.join(); } auto sleepTime = chrono::system_clock::now() - startTime; @@ -563,71 +563,71 @@ namespace UnitTest { MfTest x; AZStd::function func = AZStd::bind(&MfTest::f0, &x); - AZStd::thread(func, desc1).join(); + AZStd::thread(desc1, func).join(); func = AZStd::bind(&MfTest::f0, AZStd::ref(x)); - AZStd::thread(func, desc1).join(); + AZStd::thread(desc1, func).join(); func = AZStd::bind(&MfTest::g0, &x); - AZStd::thread(func, desc1).join(); + AZStd::thread(desc1, func).join(); func = AZStd::bind(&MfTest::g0, x); - AZStd::thread(func, desc1).join(); + AZStd::thread(desc1, func).join(); func = AZStd::bind(&MfTest::g0, AZStd::ref(x)); - AZStd::thread(func, desc1).join(); + AZStd::thread(desc1, func).join(); //// 1 - //thread( AZStd::bind(&MfTest::f1, &x, 1) , desc1).join(); - //thread( AZStd::bind(&MfTest::f1, AZStd::ref(x), 1) , desc1).join(); - //thread( AZStd::bind(&MfTest::g1, &x, 1) , desc1).join(); - //thread( AZStd::bind(&MfTest::g1, x, 1) , desc1).join(); - //thread( AZStd::bind(&MfTest::g1, AZStd::ref(x), 1) , desc1).join(); + //thread( AZStd::bind(desc1, &MfTest::f1, &x, 1)).join(); + //thread( AZStd::bind(desc1, &MfTest::f1, AZStd::ref(x), 1)).join(); + //thread( AZStd::bind(desc1, &MfTest::g1, &x, 1)).join(); + //thread( AZStd::bind(desc1, &MfTest::g1, x, 1)).join(); + //thread( AZStd::bind(desc1, &MfTest::g1, AZStd::ref(x), 1)).join(); //// 2 - //thread( AZStd::bind(&MfTest::f2, &x, 1, 2) , desc1).join(); - //thread( AZStd::bind(&MfTest::f2, AZStd::ref(x), 1, 2) , desc1).join(); - //thread( AZStd::bind(&MfTest::g2, &x, 1, 2) , desc1).join(); - //thread( AZStd::bind(&MfTest::g2, x, 1, 2) , desc1).join(); - //thread( AZStd::bind(&MfTest::g2, AZStd::ref(x), 1, 2) , desc1).join(); + //thread( AZStd::bind(desc1, &MfTest::f2, &x, 1, 2)).join(); + //thread( AZStd::bind(desc1, &MfTest::f2, AZStd::ref(x), 1, 2)).join(); + //thread( AZStd::bind(desc1, &MfTest::g2, &x, 1, 2)).join(); + //thread( AZStd::bind(desc1, &MfTest::g2, x, 1, 2)).join(); + //thread( AZStd::bind(desc1, &MfTest::g2, AZStd::ref(x), 1, 2)).join(); //// 3 - //thread( AZStd::bind(&MfTest::f3, &x, 1, 2, 3) , desc1).join(); - //thread( AZStd::bind(&MfTest::f3, AZStd::ref(x), 1, 2, 3) , desc1).join(); - //thread( AZStd::bind(&MfTest::g3, &x, 1, 2, 3) , desc1).join(); - //thread( AZStd::bind(&MfTest::g3, x, 1, 2, 3) , desc1).join(); - //thread( AZStd::bind(&MfTest::g3, AZStd::ref(x), 1, 2, 3) , desc1).join(); + //thread( AZStd::bind(desc1, &MfTest::f3, &x, 1, 2, 3)).join(); + //thread( AZStd::bind(desc1, &MfTest::f3, AZStd::ref(x), 1, 2, 3)).join(); + //thread( AZStd::bind(desc1, &MfTest::g3, &x, 1, 2, 3)).join(); + //thread( AZStd::bind(desc1, &MfTest::g3, x, 1, 2, 3)).join(); + //thread( AZStd::bind(desc1, &MfTest::g3, AZStd::ref(x), 1, 2, 3)).join(); //// 4 - //thread( AZStd::bind(&MfTest::f4, &x, 1, 2, 3, 4) , desc1).join(); - //thread( AZStd::bind(&MfTest::f4, AZStd::ref(x), 1, 2, 3, 4) , desc1).join(); - //thread( AZStd::bind(&MfTest::g4, &x, 1, 2, 3, 4) , desc1).join(); - //thread( AZStd::bind(&MfTest::g4, x, 1, 2, 3, 4) , desc1).join(); - //thread( AZStd::bind(&MfTest::g4, AZStd::ref(x), 1, 2, 3, 4) , desc1).join(); + //thread( AZStd::bind(desc1, &MfTest::f4, &x, 1, 2, 3, 4)).join(); + //thread( AZStd::bind(desc1, &MfTest::f4, AZStd::ref(x), 1, 2, 3, 4)).join(); + //thread( AZStd::bind(desc1, &MfTest::g4, &x, 1, 2, 3, 4)).join(); + //thread( AZStd::bind(desc1, &MfTest::g4, x, 1, 2, 3, 4)).join(); + //thread( AZStd::bind(desc1, &MfTest::g4, AZStd::ref(x), 1, 2, 3, 4)).join(); //// 5 - //thread( AZStd::bind(&MfTest::f5, &x, 1, 2, 3, 4, 5) , desc1).join(); - //thread( AZStd::bind(&MfTest::f5, AZStd::ref(x), 1, 2, 3, 4, 5) , desc1).join(); - //thread( AZStd::bind(&MfTest::g5, &x, 1, 2, 3, 4, 5) , desc1).join(); - //thread( AZStd::bind(&MfTest::g5, x, 1, 2, 3, 4, 5) , desc1).join(); - //thread( AZStd::bind(&MfTest::g5, AZStd::ref(x), 1, 2, 3, 4, 5) , desc1).join(); + //thread( AZStd::bind(desc1, &MfTest::f5, &x, 1, 2, 3, 4, 5)).join(); + //thread( AZStd::bind(desc1, &MfTest::f5, AZStd::ref(x), 1, 2, 3, 4, 5)).join(); + //thread( AZStd::bind(desc1, &MfTest::g5, &x, 1, 2, 3, 4, 5)).join(); + //thread( AZStd::bind(desc1, &MfTest::g5, x, 1, 2, 3, 4, 5)).join(); + //thread( AZStd::bind(desc1, &MfTest::g5, AZStd::ref(x), 1, 2, 3, 4, 5)).join(); //// 6 - //thread( AZStd::bind(&MfTest::f6, &x, 1, 2, 3, 4, 5, 6) , desc1).join(); - //thread( AZStd::bind(&MfTest::f6, AZStd::ref(x), 1, 2, 3, 4, 5, 6) , desc1).join(); - //thread( AZStd::bind(&MfTest::g6, &x, 1, 2, 3, 4, 5, 6) , desc1).join(); - //thread( AZStd::bind(&MfTest::g6, x, 1, 2, 3, 4, 5, 6) , desc1).join(); - //thread( AZStd::bind(&MfTest::g6, AZStd::ref(x), 1, 2, 3, 4, 5, 6) , desc1).join(); + //thread( AZStd::bind(desc1, &MfTest::f6, &x, 1, 2, 3, 4, 5, 6)).join(); + //thread( AZStd::bind(desc1, &MfTest::f6, AZStd::ref(x), 1, 2, 3, 4, 5, 6)).join(); + //thread( AZStd::bind(desc1, &MfTest::g6, &x, 1, 2, 3, 4, 5, 6)).join(); + //thread( AZStd::bind(desc1, &MfTest::g6, x, 1, 2, 3, 4, 5, 6)).join(); + //thread( AZStd::bind(desc1, &MfTest::g6, AZStd::ref(x), 1, 2, 3, 4, 5, 6)).join(); //// 7 - //thread( AZStd::bind(&MfTest::f7, &x, 1, 2, 3, 4, 5, 6, 7), desc1).join(); - //thread( AZStd::bind(&MfTest::f7, AZStd::ref(x), 1, 2, 3, 4, 5, 6, 7), desc1).join(); - //thread( AZStd::bind(&MfTest::g7, &x, 1, 2, 3, 4, 5, 6, 7), desc1).join(); - //thread( AZStd::bind(&MfTest::g7, x, 1, 2, 3, 4, 5, 6, 7), desc1).join(); - //thread( AZStd::bind(&MfTest::g7, AZStd::ref(x), 1, 2, 3, 4, 5, 6, 7), desc1).join(); + //thread( AZStd::bind(desc1, &MfTest::f7, &x, 1, 2, 3, 4, 5, 6, 7)).join(); + //thread( AZStd::bind(desc1, &MfTest::f7, AZStd::ref(x), 1, 2, 3, 4, 5, 6, 7)).join(); + //thread( AZStd::bind(desc1, &MfTest::g7, &x, 1, 2, 3, 4, 5, 6, 7)).join(); + //thread( AZStd::bind(desc1, &MfTest::g7, x, 1, 2, 3, 4, 5, 6, 7)).join(); + //thread( AZStd::bind(desc1, &MfTest::g7, AZStd::ref(x), 1, 2, 3, 4, 5, 6, 7)).join(); //// 8 - //thread( AZStd::bind(&MfTest::f8, &x, 1, 2, 3, 4, 5, 6, 7, 8) , desc1).join(); - //thread( AZStd::bind(&MfTest::f8, AZStd::ref(x), 1, 2, 3, 4, 5, 6, 7, 8) , desc1).join(); - //thread( AZStd::bind(&MfTest::g8, &x, 1, 2, 3, 4, 5, 6, 7, 8) , desc1).join(); - //thread( AZStd::bind(&MfTest::g8, x, 1, 2, 3, 4, 5, 6, 7, 8) , desc1).join(); - //thread( AZStd::bind(&MfTest::g8, AZStd::ref(x), 1, 2, 3, 4, 5, 6, 7, 8) , desc1).join(); + //thread( AZStd::bind(desc1, &MfTest::f8, &x, 1, 2, 3, 4, 5, 6, 7, 8)).join(); + //thread( AZStd::bind(desc1, &MfTest::f8, AZStd::ref(x), 1, 2, 3, 4, 5, 6, 7, 8)).join(); + //thread( AZStd::bind(desc1, &MfTest::g8, &x, 1, 2, 3, 4, 5, 6, 7, 8)).join(); + //thread( AZStd::bind(desc1, &MfTest::g8, x, 1, 2, 3, 4, 5, 6, 7, 8)).join(); + //thread( AZStd::bind(desc1, &MfTest::g8, AZStd::ref(x), 1, 2, 3, 4, 5, 6, 7, 8)).join(); AZ_TEST_ASSERT(x.m_hash == 1366); } diff --git a/Code/Framework/AzCore/Tests/Memory.cpp b/Code/Framework/AzCore/Tests/Memory.cpp index eb854050b6..5a483c1ed1 100644 --- a/Code/Framework/AzCore/Tests/Memory.cpp +++ b/Code/Framework/AzCore/Tests/Memory.cpp @@ -151,7 +151,7 @@ namespace UnitTest AZStd::thread m_threads[m_maxNumThreads]; for (unsigned int i = 0; i < m_maxNumThreads; ++i) { - m_threads[i] = AZStd::thread(AZStd::bind(&SystemAllocatorTest::ThreadFunc, this), &m_desc[i]); + m_threads[i] = AZStd::thread(m_desc[i], AZStd::bind(&SystemAllocatorTest::ThreadFunc, this)); // give some time offset to the threads so we can test alloc and dealloc at the same time. //AZStd::this_thread::sleep_for(AZStd::chrono::microseconds(500)); } @@ -286,7 +286,7 @@ namespace UnitTest AZStd::thread m_threads[m_maxNumThreads]; for (unsigned int i = 0; i < m_maxNumThreads; ++i) { - m_threads[i] = AZStd::thread(AZStd::bind(&SystemAllocatorTest::ThreadFunc, this), &m_desc[i]); + m_threads[i] = AZStd::thread(m_desc[i], AZStd::bind(&SystemAllocatorTest::ThreadFunc, this)); // give some time offset to the threads so we can test alloc and dealloc at the same time. AZStd::this_thread::sleep_for(AZStd::chrono::microseconds(500)); } @@ -724,7 +724,7 @@ namespace UnitTest AZStd::thread m_threads[m_maxNumThreads]; for (unsigned int i = 0; i < m_maxNumThreads; ++i) { - m_threads[i] = AZStd::thread(AZStd::bind(&ThreadPoolAllocatorTest::AllocDeallocFunc, this), &m_desc[i]); + m_threads[i] = AZStd::thread(m_desc[i], AZStd::bind(&ThreadPoolAllocatorTest::AllocDeallocFunc, this)); } for (unsigned int i = 0; i < m_maxNumThreads; ++i) @@ -743,12 +743,12 @@ namespace UnitTest for (unsigned int i = m_maxNumThreads/2; i c_str()); if (cache) diff --git a/Code/Framework/AzFramework/AzFramework/Archive/INestedArchive.h b/Code/Framework/AzFramework/AzFramework/Archive/INestedArchive.h index f85fd273ce..e89d16ee5d 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/INestedArchive.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/INestedArchive.h @@ -11,7 +11,9 @@ #include #include +#include #include +#include #include namespace AZ::IO @@ -71,6 +73,13 @@ namespace AZ::IO // multiple times FLAGS_DONT_COMPACT = 1 << 5, + // if this is set, validate header data when opening the archive + FLAGS_VALIDATE_HEADERS = 1 << 9, + + // if this is set, validate header data when opening the archive and validate CRCs when decompressing + // & reading files. + FLAGS_FULL_VALIDATE = 1 << 10, + // Disable a pak file without unloading it, this flag is used in combination with patches and multiplayer // to ensure that specific paks stay in the position(to keep the same priority) but being disabled // when running multiplayer @@ -128,6 +137,10 @@ namespace AZ::IO // Deletes all files and directories in the archive. virtual int RemoveAll() = 0; + // Summary: + // Lists all the files in the archive. + virtual int ListAllFiles(AZStd::vector& outFileEntries) = 0; + // Summary: // Finds the file; you don't have to close the returned handle. // Returns: diff --git a/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.cpp b/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.cpp index 1e0f237df5..49a44b76fa 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.cpp @@ -89,11 +89,51 @@ namespace AZ::IO return m_pCache->RemoveDir(fullPath); } + ////////////////////////////////////////////////////////////////////////// int NestedArchive::RemoveAll() { return m_pCache->RemoveAll(); } + ////////////////////////////////////////////////////////////////////////// + // Helper for 'ListAllFiles' to recursively traverse the FileEntryTree and gather all the files + void EnumerateFilesRecursive(AZ::IO::Path currentPath, ZipDir::FileEntryTree* currentTree, AZStd::vector& fileList) + { + // Drill down directories first... + for (auto dirIter = currentTree->GetDirBegin(); dirIter != currentTree->GetDirEnd(); ++dirIter) + { + if (ZipDir::FileEntryTree* subTree = currentTree->GetDirEntry(dirIter); + subTree != nullptr) + { + EnumerateFilesRecursive(currentPath / currentTree->GetDirName(dirIter), subTree, fileList); + } + } + + // Then enumerate the files in current directory... + for (auto fileIter = currentTree->GetFileBegin(); fileIter != currentTree->GetFileEnd(); ++fileIter) + { + fileList.emplace_back(currentPath / currentTree->GetFileName(fileIter)); + } + } + + ////////////////////////////////////////////////////////////////////////// + // lists all files in the archive + int NestedArchive::ListAllFiles(AZStd::vector& outFileEntries) + { + AZStd::vector filesInArchive; + + ZipDir::FileEntryTree* tree = m_pCache->GetRoot(); + if (!tree) + { + return ZipDir::ZD_ERROR_UNEXPECTED; + } + + EnumerateFilesRecursive(AZ::IO::Path{ AZ::IO::PosixPathSeparator }, tree, filesInArchive); + + AZStd::swap(outFileEntries, filesInArchive); + return ZipDir::ZD_ERROR_SUCCESS; + } + ////////////////////////////////////////////////////////////////////////// // Adds a new file to the zip or update an existing one // adds a directory (creates several nested directories if needed) diff --git a/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.h b/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.h index 34bbcdc201..59722703f2 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/NestedArchive.h @@ -39,7 +39,7 @@ namespace AZ::IO NestedArchive(IArchive* pArchive, AZStd::string_view strBindRoot, ZipDir::CachePtr pCache, uint32_t nFlags = 0); ~NestedArchive() override; - + auto GetRootFolderHandle() -> Handle override; // Adds a new file to the zip or update an existing one @@ -68,6 +68,9 @@ namespace AZ::IO // deletes all files from the archive int RemoveAll() override; + // lists all the files in the archive + int ListAllFiles(AZStd::vector& outFileEntries) override; + // finds the file; you don't have to close the returned handle Handle FindFile(AZStd::string_view szRelativePath) override; @@ -79,7 +82,6 @@ namespace AZ::IO // returns the full path to the archive file AZ::IO::PathView GetFullPath() const override; - ZipDir::Cache* GetCache(); uint32_t GetFlags() const override; bool SetFlags(uint32_t nFlagsToSet) override; @@ -87,12 +89,15 @@ namespace AZ::IO bool SetPackAccessible(bool bAccessible) override; + ZipDir::Cache* GetCache(); + protected: // returns the pointer to the relative file path to be passed // to the underlying Cache pointer. Uses the given buffer to construct the path. // returns nullptr if the file path is invalid AZ::IO::FixedMaxPathString AdjustPath(AZStd::string_view szRelativePath); + ZipDir::CachePtr m_pCache; // the binding root may be empty string - in this case, the absolute path binding won't work AZ::IO::Path m_strBindRoot; diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.cpp index d17dbd0837..13d5b0f723 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCache.cpp @@ -101,10 +101,11 @@ namespace AZ::IO::ZipDir FileEntry* operator -> () { return m_pFileEntry; } FileEntryTransactionAdd(Cache* pCache, AZStd::string_view szRelativePath) : m_pCache(pCache) + , m_szRelativePath(AZ::IO::PosixPathSeparator) , m_bCommitted(false) { // Update the cache string pool with the relative path to the file - auto pathIt = m_pCache->m_relativePathPool.emplace(AZ::IO::PathView(szRelativePath).LexicallyNormal()); + auto pathIt = m_pCache->m_relativePathPool.emplace(AZ::IO::PathView(szRelativePath, AZ::IO::PosixPathSeparator).LexicallyNormal()); m_szRelativePath = *pathIt.first; // this is the name of the directory - create it or find it m_pFileEntry = m_pCache->GetRoot()->Add(m_szRelativePath.Native()); @@ -740,6 +741,16 @@ namespace AZ::IO::ZipDir { return ZD_ERROR_CORRUPTED_DATA; } + if (pFileEntry->bCheckCRCNextRead) + { + pFileEntry->bCheckCRCNextRead = false; + uLong uCRC32 = AZ::Crc32((Bytef*)pUncompressed, nSizeUncompressed); + if (uCRC32 != pFileEntry->desc.lCRC32) + { + AZ_Warning("Archive", false, "ZD_ERROR_CRC32_CHECK: Uncompressed stream CRC32 check failed"); + return ZD_ERROR_CRC32_CHECK; + } + } } } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.cpp index 5c5e93d441..d5bd4d2840 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.cpp @@ -29,7 +29,7 @@ namespace AZ::IO::ZipDir // this sets the window size of the blocks of data read from the end of the file to find the Central Directory Record // since normally there are no static constexpr size_t CDRSearchWindowSize = 0x100; - CacheFactory::CacheFactory(InitMethodEnum nInitMethod, uint32_t nFlags) + CacheFactory::CacheFactory(InitMethod nInitMethod, uint32_t nFlags) { m_nCDREndPos = 0; m_bBuildFileEntryMap = false; // we only need it for validation/debugging @@ -448,7 +448,6 @@ namespace AZ::IO::ZipDir // builds up the m_mapFileEntries bool CacheFactory::BuildFileEntryMap() { - Seek(m_CDREnd.lCDROffset); if (m_CDREnd.lCDRSize == 0) @@ -530,14 +529,6 @@ namespace AZ::IO::ZipDir { // Add this file entry. char* str = reinterpret_cast(pFileName); - for (int i = 0; i < pFile->nFileNameLength; i++) - { - str[i] = std::tolower(str[i], std::locale()); - if (str[i] == AZ_WRONG_FILESYSTEM_SEPARATOR) - { - str[i] = AZ_CORRECT_FILESYSTEM_SEPARATOR; - } - } str[pFile->nFileNameLength] = 0; // Not standard!, may overwrite signature of the next memory record data in zip. AddFileEntry(str, pFile, extra); } @@ -574,11 +565,7 @@ namespace AZ::IO::ZipDir FileEntryBase fileEntry(*pFileHeader, extra); - // when using encrypted headers we should always initialize data offsets from CDR - if ((m_encryptedHeaders != ZipFile::HEADERS_NOT_ENCRYPTED || m_nInitMethod >= ZD_INIT_FULL) && pFileHeader->desc.lSizeCompressed) - { - InitDataOffset(fileEntry, pFileHeader); - } + InitDataOffset(fileEntry, pFileHeader); if (m_bBuildFileEntryMap) { @@ -606,142 +593,81 @@ namespace AZ::IO::ZipDir { Seek(pFileHeader->lLocalHeaderOffset); - // read the local file header and the name (for validation) into the buffer - AZStd::vectorpBuffer; - uint32_t nBufferLength = sizeof(ZipFile::LocalFileHeader) + pFileHeader->nFileNameLength; - pBuffer.resize(nBufferLength); - Read(&pBuffer[0], nBufferLength); - - // validate the local file header (compare with the CDR file header - they should contain basically the same information) - const auto* pLocalFileHeader = reinterpret_cast(&pBuffer[0]); - if (pFileHeader->desc != pLocalFileHeader->desc - || pFileHeader->nMethod != pLocalFileHeader->nMethod - || pFileHeader->nFileNameLength != pLocalFileHeader->nFileNameLength - // for a tough validation, we can compare the timestamps of the local and central directory entries - // but we won't do that for backward compatibility with ZipDir - //|| pFileHeader->nLastModDate != pLocalFileHeader->nLastModDate - //|| pFileHeader->nLastModTime != pLocalFileHeader->nLastModTime - ) - { - AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED:" - " The local file header descriptor doesn't match the basic parameters declared in the global file header in the file." - " The archive content is misconsistent and may be damaged. Please try to repair the archive"); - return; - } + // Read only the LocalFileHeader w/ no additional bytes ('name' or 'extra' fields) + AZStd::vector buffer; + uint32_t bufferLen = sizeof(ZipFile::LocalFileHeader); + buffer.resize_no_construct(bufferLen); + Read(buffer.data(), bufferLen); - // now compare the local file name with the one recorded in CDR: they must match. - auto CompareNoCase = [](const char lhs, const char rhs) { return std::tolower(lhs, std::locale()) == std::tolower(rhs, std::locale()); }; - auto zipFileDataBegin = pBuffer.begin() + sizeof(ZipFile::LocalFileHeader); - auto zipFileDataEnd = zipFileDataBegin + pFileHeader->nFileNameLength; - if (!AZStd::equal(zipFileDataBegin, zipFileDataEnd, reinterpret_cast(pFileHeader + 1), CompareNoCase)) - { - // either file name, or the extra field do not match - AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED:" - " The local file header contains file name which does not match the file name of the global file header." - " The archive content is misconsistent with its directory. Please repair the archive"); - return; - } + const auto* localFileHeader = reinterpret_cast(buffer.data()); - fileEntry.nFileDataOffset = pFileHeader->lLocalHeaderOffset + sizeof(ZipFile::LocalFileHeader) + pLocalFileHeader->nFileNameLength + pLocalFileHeader->nExtraFieldLength; - } + // set the correct file data offset... + fileEntry.nFileDataOffset = pFileHeader->lLocalHeaderOffset + sizeof(ZipFile::LocalFileHeader) + + localFileHeader->nFileNameLength + localFileHeader->nExtraFieldLength; - // make sure it's the same file and the fileEntry structure is properly initialized - AZ_Assert(fileEntry.nFileHeaderOffset == pFileHeader->lLocalHeaderOffset, "The file entry header offset doesn't match the file header local offst"); - - fileEntry.nEOFOffset = fileEntry.nFileDataOffset + fileEntry.desc.lSizeCompressed; - - if (fileEntry.nFileDataOffset >= m_nCDREndPos) - { - AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED:" - " The global file header declares the file which crosses the boundaries of the archive." - " The archive is either corrupted or truncated, please try to repair it"); - return; - } + fileEntry.nEOFOffset = fileEntry.nFileDataOffset + fileEntry.desc.lSizeCompressed; - if (m_nInitMethod >= ZD_INIT_VALIDATE) - { - Validate(fileEntry); - } - } + if (m_nInitMethod != ZipDir::InitMethod::Default) + { + if (m_nInitMethod == ZipDir::InitMethod::FullValidation) + { + // Mark the FileEntry to check CRC when the next read occurs + fileEntry.bCheckCRCNextRead = true; + } - ////////////////////////////////////////////////////////////////////////// - // reads the file pointed by the given header and entry (they must be coherent) - // and decompresses it; then calculates and validates its CRC32 - void CacheFactory::Validate(const FileEntryBase& fileEntry) - { - AZStd::vector pBuffer; - // validate the file contents - // allocate memory for both the compressed data and uncompressed data - pBuffer.resize(fileEntry.desc.lSizeCompressed + fileEntry.desc.lSizeUncompressed); - char* pUncompressed = &pBuffer[fileEntry.desc.lSizeCompressed]; - char* pCompressed = &pBuffer[0]; + // Timestamps + if (pFileHeader->nLastModDate != localFileHeader->nLastModDate + || pFileHeader->nLastModTime != localFileHeader->nLastModTime) + { + AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED: (%s)\n" + " The local file header's modification timestamps don't match that of the global file header in the archive." + " The archive timestamps are inconsistent and may be damaged. Check the archive file.", m_szFilename.c_str()); + // don't return here, it may be ok. + } - AZ_Assert(fileEntry.nFileDataOffset != FileEntry::INVALID_DATA_OFFSET, "File entry has invalid data offset of %" PRIx32, FileEntry::INVALID_DATA_OFFSET); - Seek(fileEntry.nFileDataOffset); + // Validate data + if (pFileHeader->desc != localFileHeader->desc // this checks CRCs and compressed/uncompressed sizes + || pFileHeader->nMethod != localFileHeader->nMethod + || pFileHeader->nFileNameLength != localFileHeader->nFileNameLength) + { + AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED: (%s)\n" + " The local file header descriptor doesn't match basic parameters declared in the global file header in the file." + " The archive content is inconsistent and may be damaged. Please try to repair the archive.", m_szFilename.c_str()); + // return here because further checks aren't worse than this. + return; + } - Read(pCompressed, fileEntry.desc.lSizeCompressed); + // Read extra data + uint32_t extraDataLen = localFileHeader->nFileNameLength + localFileHeader->nExtraFieldLength; + buffer.resize_no_construct(buffer.size() + extraDataLen); + Read(buffer.data() + buffer.size(), extraDataLen); - size_t nDestSize = fileEntry.desc.lSizeUncompressed; - int nError = Z_OK; - if (fileEntry.nMethod) - { - nError = ZipRawUncompress(pUncompressed, &nDestSize, pCompressed, fileEntry.desc.lSizeCompressed); - } - else - { - AZ_Assert(fileEntry.desc.lSizeCompressed == fileEntry.desc.lSizeUncompressed, "Uncompressed file does not have the same commpressed %u and uncompressed file sizes %u", - fileEntry.desc.lSizeCompressed, fileEntry.desc.lSizeUncompressed); - memcpy(pUncompressed, pCompressed, fileEntry.desc.lSizeUncompressed); - } - switch (nError) - { - case Z_OK: - break; - case Z_MEM_ERROR: - AZ_Warning("Archive", false, "ZD_ERROR_ZLIB_NO_MEMORY: ZLib reported out-of-memory error"); - return; - case Z_BUF_ERROR: - AZ_Warning("Archive", false, "ZD_ERROR_ZLIB_CORRUPTED_DATA: ZLib reported compressed stream buffer error"); - return; - case Z_DATA_ERROR: - AZ_Warning("Archive", false, "ZD_ERROR_ZLIB_CORRUPTED_DATA: ZLib reported compressed stream data error"); - return; - default: - AZ_Warning("Archive", false, "ZD_ERROR_ZLIB_FAILED: ZLib reported an unexpected unknown error"); - return; - } + // Compare local file name with the CDR file name, they should match + AZStd::string_view zipFileName{ buffer.data() + sizeof(ZipFile::LocalFileHeader), localFileHeader->nFileNameLength }; + AZStd::string_view cdrFileName{ reinterpret_cast(pFileHeader + 1), pFileHeader->nFileNameLength }; + if (zipFileName != cdrFileName) + { + AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED: (%s)\n" + " The file name in the local file header doesn't match the name in the global file header." + " The archive content is inconsisten with the directory. Please check the archive.", m_szFilename.c_str()); + } - if (nDestSize != fileEntry.desc.lSizeUncompressed) - { - AZ_Warning("Archive", false, "ZD_ERROR_CORRUPTED_DATA: Uncompressed stream doesn't match the size of uncompressed file stored in the archive file headers"); - return; - } + // CDR and local "extra field" lengths may be different, should we compare them if they are equal? - uLong uCRC32 = AZ::Crc32((Bytef*)pUncompressed, nDestSize); - if (uCRC32 != fileEntry.desc.lCRC32) - { - AZ_Warning("Archive", false, "ZD_ERROR_CRC32_CHECK: Uncompressed stream CRC32 check failed"); - return; - } - } + // make sure it's the same file and the fileEntry structure is properly initialized + AZ_Assert(fileEntry.nFileHeaderOffset == pFileHeader->lLocalHeaderOffset, + "The file entry header offset doesn't match the file header local offst (%s)", m_szFilename.c_str()); + if (fileEntry.nFileDataOffset >= m_nCDREndPos) + { + AZ_Warning("Archive", false, "ZD_ERROR_VALIDATION_FAILED: (%s)\n" + " The global file header declares the file which crosses the boundaries of the archive." + " The archive is either corrupted or truncated, please try to repair it", m_szFilename.c_str()); + } - ////////////////////////////////////////////////////////////////////////// - // extracts the file path from the file header with subsequent information - // may, or may not, put all letters to lower-case (depending on whether the system is to be case-sensitive or not) - // it's the responsibility of the caller to ensure that the file name is in readable valid memory - char* CacheFactory::GetFilePath(const char* pFileName, uint16_t nFileNameLength) - { - static char strResult[AZ_MAX_PATH_LEN]; - AZ_Assert(nFileNameLength < AZ_MAX_PATH_LEN, "Only filenames shorter than %zu can be copied from filename parameter", AZ_MAX_PATH_LEN); - memcpy(strResult, pFileName, nFileNameLength); - strResult[nFileNameLength] = 0; - for (int i = 0; i < nFileNameLength; i++) - { - strResult[i] = std::tolower(strResult[i], std::locale{}); + // End Validation + } } - - return strResult; } // seeks in the file relative to the starting position diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.h b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.h index c31d4d7dfd..1612829f13 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirCacheFactory.h @@ -39,7 +39,7 @@ namespace AZ::IO::ZipDir // initializes the internal structures // nFlags can have FLAGS_READ_ONLY flag, in this case the object will be opened only for reading - CacheFactory(InitMethodEnum nInitMethod, uint32_t nFlags = 0); + CacheFactory(InitMethod nInitMethod, uint32_t nFlags = 0); ~CacheFactory(); // the new function creates a new cache @@ -66,28 +66,6 @@ namespace AZ::IO::ZipDir // This function can actually modify strFilePath variable, make sure you use a copy of the real path. void AddFileEntry(char* strFilePath, const ZipFile::CDRFileHeader* pFileHeader, const SExtraZipFileData& extra);// throw (ErrorEnum); - // extracts the file path from the file header with subsequent information - // may, or may not, put all letters to lower-case (depending on whether the system is to be case-sensitive or not) - // it's the responsibility of the caller to ensure that the file name is in readable valid memory - char* GetFilePath(const ZipFile::CDRFileHeader* pFileHeader) - { - return GetFilePath((const char*)(pFileHeader + 1), pFileHeader->nFileNameLength); - } - // extracts the file path from the file header with subsequent information - // may, or may not, put all letters to lower-case (depending on whether the system is to be case-sensitive or not) - // it's the responsibility of the caller to ensure that the file name is in readable valid memory - char* GetFilePath(const ZipFile::LocalFileHeader* pFileHeader) - { - return GetFilePath((const char*)(pFileHeader + 1), pFileHeader->nFileNameLength); - } - // extracts the file path from the file header with subsequent information - // may, or may not, put all letters to lower-case (depending on whether the system is to be case-sensitive or not) - // it's the responsibility of the caller to ensure that the file name is in readable valid memory - char* GetFilePath(const char* pFileName, uint16_t nFileNameLength); - - // validates (if the init method has the corresponding value) the given file/header - void Validate(const FileEntryBase& fileEntry); - // initializes the actual data offset in the file in the fileEntry structure // searches to the local file header, reads it and calculates the actual offset in the file void InitDataOffset(FileEntryBase& fileEntry, const ZipFile::CDRFileHeader* pFileHeader); @@ -104,7 +82,7 @@ namespace AZ::IO::ZipDir AZStd::string m_szFilename; CZipFile m_fileExt; - InitMethodEnum m_nInitMethod; + InitMethod m_nInitMethod; uint32_t m_nFlags; ZipFile::CDREnd m_CDREnd; @@ -129,7 +107,7 @@ namespace AZ::IO::ZipDir ZipFile::CryCustomEncryptionHeader m_headerEncryption; ZipFile::CrySignedCDRHeader m_headerSignature; ZipFile::CryCustomExtendedHeader m_headerExtended; - }; + } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirList.cpp b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirList.cpp index 729f394b9d..ab9c356d7f 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirList.cpp +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirList.cpp @@ -68,14 +68,14 @@ namespace AZ::IO::ZipDir { for (FileEntryTree::SubdirMap::iterator it = pTree->GetDirBegin(); it != pTree->GetDirEnd(); ++it) { - AddAllFiles(it->second.get(), (AZ::IO::Path(strRoot) / it->first).Native()); + AddAllFiles(it->second.get(), (AZ::IO::Path(strRoot, AZ::IO::PosixPathSeparator) / it->first).Native()); } for (FileEntryTree::FileMap::iterator it = pTree->GetFileBegin(); it != pTree->GetFileEnd(); ++it) { FileRecord rec; rec.pFileEntryBase = pTree->GetFileEntry(it); - rec.strPath = (AZ::IO::Path(strRoot) / it->first).Native(); + rec.strPath = (AZ::IO::Path(strRoot, AZ::IO::PosixPathSeparator) / it->first).Native(); push_back(rec); } } diff --git a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.h b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.h index 9295a7dd95..c890d498e4 100644 --- a/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.h +++ b/Code/Framework/AzFramework/AzFramework/Archive/ZipDirStructures.h @@ -119,19 +119,28 @@ namespace AZ::IO::ZipDir const char* m_szDescription; }; +#if defined(_RELEASE) + inline static constexpr bool IsReleaseConfig{ true }; +#else + inline static constexpr bool IsReleaseConfig{}; +#endif // _RELEASE + // possible initialization methods - enum InitMethodEnum + enum class InitMethod { - // initialize as fast as possible, with minimal validation - ZD_INIT_FAST, - // after initialization, scan through all file headers, precache the actual file data offset values and validate the headers - ZD_INIT_FULL, - // scan all file headers and try to decompress the data, searching for corrupted files - ZD_INIT_VALIDATE_IN_MEMORY, - // store archive in memory - ZD_INIT_VALIDATE, - // maximum level of validation, checks for integrity of the archive - ZD_INIT_VALIDATE_MAX = ZD_INIT_VALIDATE + // initializes without any sort of extra validation steps + Default, + + // initializes with extra validation steps + // not available in RELEASE + // will check CDR and local headers data match + ValidateHeaders, + + // initializes with extra validation steps + // not available in RELEASE + // will check CDR and local headers data match + // will check file data CRC matches (when file is read) + FullValidation, }; // Uncompresses raw (without wrapping) data that is compressed with method 8 (deflated) in the Zip file @@ -184,7 +193,11 @@ namespace AZ::IO::ZipDir // the offset to the start of the next file's header - this // can be used to calculate the available space in zip file uint32_t nEOFOffset{}; + + // whether to check the CRC upon the next data read + bool bCheckCRCNextRead{}; }; + // this is the record about the file in the Zip file. struct FileEntry : FileEntryBase diff --git a/Code/Framework/AzFramework/AzFramework/Network/AssetProcessorConnection.cpp b/Code/Framework/AzFramework/AzFramework/Network/AssetProcessorConnection.cpp index 2e1cde443c..74b92a5681 100644 --- a/Code/Framework/AzFramework/AzFramework/Network/AssetProcessorConnection.cpp +++ b/Code/Framework/AzFramework/AzFramework/Network/AssetProcessorConnection.cpp @@ -593,7 +593,7 @@ namespace AzFramework DebugMessage("StartThread: Starting %s", thread.m_desc.m_name); thread.m_join = false; - thread.m_thread = AZStd::thread(thread.m_main, &thread.m_desc); + thread.m_thread = AZStd::thread(thread.m_desc, thread.m_main); } void AssetProcessorConnection::JoinThread(ThreadState& thread, AZStd::condition_variable* wakeUpCondition /* = nullptr */) diff --git a/Code/Framework/AzFramework/AzFramework/TargetManagement/TargetManagementComponent.cpp b/Code/Framework/AzFramework/AzFramework/TargetManagement/TargetManagementComponent.cpp index 1c26cbc744..8e6bacda2b 100644 --- a/Code/Framework/AzFramework/AzFramework/TargetManagement/TargetManagementComponent.cpp +++ b/Code/Framework/AzFramework/AzFramework/TargetManagement/TargetManagementComponent.cpp @@ -319,7 +319,7 @@ namespace AzFramework AZStd::thread_desc td; td.m_name = "TargetManager Thread"; td.m_cpuId = AFFINITY_MASK_USERTHREADS; - m_threadHandle = AZStd::thread(AZStd::bind(&TargetManagementComponent::TickThread, this), &td); + m_threadHandle = AZStd::thread(td, AZStd::bind(&TargetManagementComponent::TickThread, this)); } void TargetManagementComponent::Deactivate() diff --git a/Code/Framework/AzNetworking/AzNetworking/Utilities/TimedThread.cpp b/Code/Framework/AzNetworking/AzNetworking/Utilities/TimedThread.cpp index c85d15ae45..565b37202a 100644 --- a/Code/Framework/AzNetworking/AzNetworking/Utilities/TimedThread.cpp +++ b/Code/Framework/AzNetworking/AzNetworking/Utilities/TimedThread.cpp @@ -26,27 +26,29 @@ namespace AzNetworking { m_running = true; m_joinable = true; - m_thread = AZStd::thread([this]() - { - OnStart(); - while (m_running) + m_thread = AZStd::thread( + m_threadDesc, + [this]() { - const AZ::TimeMs startTimeMs = AZ::GetElapsedTimeMs(); - OnUpdate(m_updateRate); - const AZ::TimeMs updateTimeMs = AZ::GetElapsedTimeMs() - startTimeMs; - - if (m_updateRate > updateTimeMs) + OnStart(); + while (m_running) { - AZStd::chrono::milliseconds sleepTimeMs(static_cast(m_updateRate - updateTimeMs)); - AZStd::this_thread::sleep_for(sleepTimeMs); - } - else if (m_updateRate < updateTimeMs) - { - AZLOG(NET_TimedThread, "TimedThread bled %d ms", aznumeric_cast(updateTimeMs - m_updateRate)); + const AZ::TimeMs startTimeMs = AZ::GetElapsedTimeMs(); + OnUpdate(m_updateRate); + const AZ::TimeMs updateTimeMs = AZ::GetElapsedTimeMs() - startTimeMs; + + if (m_updateRate > updateTimeMs) + { + AZStd::chrono::milliseconds sleepTimeMs(static_cast(m_updateRate - updateTimeMs)); + AZStd::this_thread::sleep_for(sleepTimeMs); + } + else if (m_updateRate < updateTimeMs) + { + AZLOG(NET_TimedThread, "TimedThread bled %d ms", aznumeric_cast(updateTimeMs - m_updateRate)); + } } - } - OnStop(); - }, &m_threadDesc); + OnStop(); + }); } void TimedThread::Stop() diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveAPI.h b/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveAPI.h index fa8c2c14ff..11c4c85722 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveAPI.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveAPI.h @@ -12,95 +12,84 @@ #include #include #include +#include namespace AzToolsFramework { - // use bind if you need additional context. - // Parameters: - // bool - If the archive command was successful or not. - typedef AZStd::function ArchiveResponseCallback; - // bool - If the archive command was successful or not. - // AZStd::string - The console output from the command. - typedef AZStd::function ArchiveResponseOutputCallback; - - //! ArchiveCommands //! This bus handles messages relating to archive commands //! archive commands are ASYNCHRONOUS //! archive formats officially supported are .zip - //! do not block the main thread waiting for a response, it is not okay - //! you will not get a message delivered unless you tick the tickbus anyway! class ArchiveCommands : public AZ::EBusTraits { public: - - using Bus = AZ::EBus; - + // EBus Traits static const AZ::EBusHandlerPolicy HandlerPolicy = AZ::EBusHandlerPolicy::Single; static const AZ::EBusAddressPolicy AddressPolicy = AZ::EBusAddressPolicy::Single; typedef AZStd::recursive_mutex MutexType; static const bool LocklessDispatch = true; - virtual ~ArchiveCommands() {} - - //! Start an async task to extract an archive to the target directory - //! taskHandles are used to cancel a task at some point in the future and are provided by the caller per task. - //! Multiple tasks can be associated with the same handle - virtual void ExtractArchive(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseCallback& respCallback) = 0; - virtual void ExtractArchiveOutput(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) = 0; - // Maintaining backwards API compatibility - ExtractArchiveBlocking below passes in extractWithRoot as an option - virtual void ExtractArchiveWithoutRoot(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) = 0; - - //! Start a sync task to extract an archive to the target directory - //! If you do not want to extract the root folder then set extractWithRootDirectory to false. - virtual bool ExtractArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& destinationPath, bool extractWithRootDirectory) = 0; - - //! Extract a single file asynchronously from the archive to the destination. - //! Uses cwd if destinationPath empty. overWrite = true for overwrite existing files, false for skipExisting - //! taskHandles are used to cancel a task at some point in the future and are provided by the caller per task. - //! Multiple tasks can be associated with the same handle - virtual void ExtractFile(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) = 0; - - //! Extract a single file from the archive to the destination and block until finished. - //! Uses cwd if destinationPath empty. overWrite = true for overwrite existing files, false for skipExisting - virtual bool ExtractFileBlocking(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite) = 0; - //! Start an async task to create an archive of the target directory (recursively) - //! taskHandles are used to cancel a task at some point in the future and are provided by the caller per task. - //! Multiple tasks can be associated with the same handle. - virtual void CreateArchive(const AZStd::string& archivePath, const AZStd::string& dirToArchive, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) = 0; - - //! Start a sync task to create an archive of the target directory (recursively) - virtual bool CreateArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& dirToArchive) = 0; - - //! Start an async task to retrieve the list of files and their relative paths within an archive (recursively) - //! taskHandles are used to cancel a task at some point in the future and are provided by the caller per task. - //! Multiple tasks can be associated with the same handle. - virtual void ListFilesInArchive(const AZStd::string& archivePath, AZStd::vector& fileEntries, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) = 0; - - //! Start a sync task to retrieve the list of files and their relative paths within an archive (recursively) - virtual bool ListFilesInArchiveBlocking(const AZStd::string& archivePath, AZStd::vector& fileEntries) = 0; - - //! Start an async task to add a file to a preexisting archive. - //! fileToAdd must be a relative path to the file from the working directory. The path to the file from the root of the archive will be the same as the relative path to the file on disk. - //! taskHandles are used to cancel a task at some point in the future and are provided by the caller per task. - //! Multiple tasks can be associated with the same handle. - virtual void AddFileToArchive(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& fileToAdd, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) = 0; - - //! Start a sync task to add a file to a preexisting archive. - //! fileToAdd must be a relative path to the file from the working directory. The path to the file from the root of the archive will be the same as the relative path to the file on disk. - virtual bool AddFileToArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& fileToAdd) = 0; - - //! Start an async task to add files to a archive. - //! File paths inside the list file must either be a relative path from the working directory or an absolute path. - virtual void AddFilesToArchive(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& listFilePath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) = 0; - - //! Start a sync task to add files to an archive. - //! File paths inside the list file must either be a relative path from the working directory or an absolute path. - virtual bool AddFilesToArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& listFilePath) = 0; - - //! Cancels tasks associtated with the given handle. Blocks until all tasks are cancelled. - virtual void CancelTasks(AZ::Uuid taskHandle) = 0; + virtual ~ArchiveCommands() = default; + + //! Create an archive of the target directory (all files and subdirectories) + //! @param archivePath The path of the archive to create + //! @dirToArchive The directory to be added to the archive + //! @return Future (bool) which can obtain the success value of the operation + [[nodiscard]] virtual std::future CreateArchive( + const AZStd::string& archivePath, + const AZStd::string& dirToArchive) = 0; + + //! Extract an archive to the target directory + //! @param archivePath The path of the archive to extract + //! @param destinationPath The directory where files will be extracted to + //! @return Future (bool) which can obtain the success value of the operation + [[nodiscard]] virtual std::future ExtractArchive( + const AZStd::string& archivePath, + const AZStd::string& destinationPath) = 0; + + //! Extract a single file from the archive to the destination + //! Destination path should not be empty + //! @param archivePath The path of the archive to extract from + //! @param fileInArchive A path to a file, relative to root of archive + //! @param destinationPath The directory where file will be extracted to + //! @return Future (bool) which can obtain the success value of the operation + [[nodiscard]] virtual std::future ExtractFile( + const AZStd::string& archivePath, + const AZStd::string& fileInArchive, + const AZStd::string& destinationPath) = 0; + + //! Retrieve the list of files contained in an archive (all files and subdirectories) + //! @param archivePath The path of the archive to list + //! @param outFileEntries An out parameter that will contain the file paths found + //! @return True if successful, false otherwise + virtual bool ListFilesInArchive(const AZStd::string& archivePath, AZStd::vector& outFileEntries) = 0; + + //! Add a file to an archive + //! The archive might not exist yet + //! The file path relative to the working directory will be replicated in the archive + //! @param archivePath The path of the archive to add to + //! @param workingDirectory A directory that will be the starting path of the file to be added + //! @param fileToAdd A path to the file relative to the working directory + //! @return Future (bool) which can obtain the success value of the operation + [[nodiscard]] virtual std::future AddFileToArchive( + const AZStd::string& archivePath, + const AZStd::string& workingDirectory, + const AZStd::string& fileToAdd) = 0; + + //! Add files to an archive provided from a file listing + //! The archive might not exist yet + //! File paths in the file list should be relative to root of the archive + //! @param archivePath The path of the archive to add to + //! @param workingDirectory A directory that will be the starting path of the list of files to add + //! @param listFilePath Full path to a text file that contains the list of files to add + //! @return Future (bool) which can obtain the success value of the operation + [[nodiscard]] virtual std::future AddFilesToArchive( + const AZStd::string& archivePath, + const AZStd::string& workingDirectory, + const AZStd::string& listFilePath) = 0; }; + using ArchiveCommandsBus = AZ::EBus; + }; // namespace AzToolsFramework diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveComponent.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveComponent.cpp index 5004108132..8e07794c33 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveComponent.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveComponent.cpp @@ -12,118 +12,98 @@ #include #include -#include - +#include +#include #include #include #include + namespace AzToolsFramework { - // Forward declare platform specific functions - namespace Platform - { - AZStd::string GetZipExePath(); - AZStd::string GetUnzipExePath(); - - AZStd::string GetCreateArchiveCommand(const AZStd::string& archivePath, const AZStd::string& dirToArchive); - AZStd::string GetExtractArchiveCommand(const AZStd::string& archivePath, const AZStd::string& destinationPath, bool includeRoot); - AZStd::string GetAddFileToArchiveCommand(const AZStd::string& archivePath, const AZStd::string& file); - AZStd::string GetAddFilesToArchiveCommand(const AZStd::string& archivePath, const AZStd::string& listFilePath); - AZStd::string GetExtractFileCommand(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite); - AZStd::string GetListFilesInArchiveCommand(const AZStd::string& archivePath); - void ParseConsoleOutputFromListFilesInArchive(const AZStd::string& consoleOutput, AZStd::vector& fileEntries); - } + constexpr const char s_traceName[] = "ArchiveComponent"; + constexpr AZ::u32 s_compressionMethod = AZ::IO::INestedArchive::METHOD_DEFLATE; + constexpr AZ::s32 s_compressionLevel = AZ::IO::INestedArchive::LEVEL_NORMAL; + constexpr CompressionCodec::Codec s_compressionCodec = CompressionCodec::Codec::ZLIB; - const char s_traceName[] = "ArchiveComponent"; - const unsigned int g_sleepDuration = 1; - - // Echoes all results of stdout and stderr to console and never blocks - class ConsoleEchoCommunicator + namespace ArchiveUtils { - public: - ConsoleEchoCommunicator(AzFramework::ProcessCommunicator* communicator) - : m_communicator(communicator) + // Read a file's contents into a provided buffer. + // Does not add a zero byte at the end of the buffer. + // returns true if read was successful, false otherwise. + bool ReadFile(const AZ::IO::Path& filePath, AZ::IO::OpenMode openMode, AZStd::vector& outBuffer) { - } - - ~ConsoleEchoCommunicator() - { - } + auto fileIO = AZ::IO::FileIOBase::GetDirectInstance(); + if (!fileIO) + { + return false; + } - // Call this periodically to drain the buffers - void Pump() - { - if (m_communicator->IsValid()) + bool success = false; + AZ::IO::HandleType fileHandle = AZ::IO::InvalidHandle; + if (fileIO->Open(filePath.c_str(), openMode, fileHandle)) { - AZ::u32 readBufferSize = 0; - AZStd::string readBuffer; - // Don't call readOutput unless there is output or else it will block... - readBufferSize = m_communicator->PeekOutput(); - if (readBufferSize) - { - readBuffer.resize_no_construct(readBufferSize + 1); - readBuffer[readBufferSize] = '\0'; - m_communicator->ReadOutput(readBuffer.data(), readBufferSize); - EchoBuffer(readBuffer); - } - readBufferSize = m_communicator->PeekError(); - if (readBufferSize) + AZ::u64 fileSize = 0; + if (fileIO->Size(fileHandle, fileSize) && fileSize != 0) { - readBuffer.resize_no_construct(readBufferSize + 1); - readBuffer[readBufferSize] = '\0'; - m_communicator->ReadError(readBuffer.data(), readBufferSize); - EchoBuffer(readBuffer); + outBuffer.resize_no_construct(fileSize); + + AZ::u64 bytesRead = 0; + if (fileIO->Read(fileHandle, outBuffer.data(), fileSize, true, &bytesRead)) + { + success = (fileSize == bytesRead); + } } + + fileIO->Close(fileHandle); } + + return success; } - private: - void EchoBuffer(const AZStd::string& buffer) + // Reads a text file that contains a list of file paths. + // Tokenize the file by lines. + // Calls the lineVisitor function for each line of the file. + void ProcessFileList(const AZ::IO::Path& filePath, AZStd::function lineVisitor) { - size_t startIndex = 0; - size_t endIndex = 0; - const size_t bufferSize = buffer.size(); - for (size_t i = 0; i < bufferSize; ++i) + AZStd::vector fileBuffer; + if (ReadFile(filePath, AZ::IO::OpenMode::ModeText | AZ::IO::OpenMode::ModeRead, fileBuffer)) { - if (buffer[i] == '\n' || buffer[i] == '\0') - { - endIndex = i; - bool isEmptyMessage = (endIndex - startIndex == 1) && (buffer[startIndex] == '\r'); - if (!isEmptyMessage) - { - AZ_Printf(s_traceName, "%s", buffer.substr(startIndex, endIndex - startIndex).c_str()); - } - startIndex = endIndex + 1; - } + AZ::StringFunc::TokenizeVisitor(AZStd::string_view{ fileBuffer.data(), fileBuffer.size() }, lineVisitor, "\n"); } } - AzFramework::ProcessCommunicator* m_communicator = nullptr; - }; + } // namespace ArchiveUtils void ArchiveComponent::Activate() { - m_zipExePath = Platform::GetZipExePath(); - m_unzipExePath = Platform::GetUnzipExePath(); + m_fileIO = AZ::IO::FileIOBase::GetDirectInstance(); + if (m_fileIO == nullptr) + { + AZ_Error(s_traceName, false, "Failed to create a LocalFileIO instance!"); + } - ArchiveCommands::Bus::Handler::BusConnect(); + m_archive = AZ::Interface::Get(); + if (m_archive == nullptr) + { + AZ_Error(s_traceName, false, "Failed to get IArchive interface!"); + } + + ArchiveCommandsBus::Handler::BusConnect(); } void ArchiveComponent::Deactivate() { - ArchiveCommands::Bus::Handler::BusDisconnect(); + ArchiveCommandsBus::Handler::BusDisconnect(); - AZStd::unique_lock lock(m_threadControlMutex); - for (auto pair : m_threadInfoMap) + m_fileIO = nullptr; + m_archive = nullptr; + + for (AZStd::thread& t : m_threads) { - ThreadInfo& info = pair.second; - info.shouldStop = true; - m_cv.wait(lock, [&info]() { - return info.threads.size() == 0; - }); + t.join(); } - m_threadInfoMap.clear(); + m_threads = {}; } void ArchiveComponent::Reflect(AZ::ReflectContext * context) @@ -132,7 +112,7 @@ namespace AzToolsFramework { serializeContext->Class() ->Version(2) - ->Attribute(AZ::Edit::Attributes::SystemComponentTags, AZStd::vector({ AZ_CRC("AssetBuilder", 0xc739c7d7) })) + ->Attribute(AZ::Edit::Attributes::SystemComponentTags, AZStd::vector({ AZ_CRC_CE("AssetBuilder") })) ; if (AZ::EditContext* editContext = serializeContext->GetEditContext()) @@ -141,320 +121,480 @@ namespace AzToolsFramework "Archive", "Handles creation and extraction of zip archives.") ->ClassElement(AZ::Edit::ClassElements::EditorData, "") ->Attribute(AZ::Edit::Attributes::Category, "Editor") - ->Attribute(AZ::Edit::Attributes::AppearsInAddComponentMenu, AZ_CRC("System", 0xc94d118b)) + ->Attribute(AZ::Edit::Attributes::AppearsInAddComponentMenu, AZ_CRC_CE("System")) ; } } } - void ArchiveComponent::CreateArchive(const AZStd::string& archivePath, const AZStd::string& dirToArchive, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) + std::future ArchiveComponent::CreateArchive( + const AZStd::string& archivePath, + const AZStd::string& dirToArchive) { - AZStd::string commandLineArgs = AZStd::string::format(R"(a -tzip -mx=1 "%s" -r "%s\*")", archivePath.c_str(), dirToArchive.c_str()); - LaunchZipExe(m_zipExePath, commandLineArgs, respCallback, taskHandle); - } - - bool ArchiveComponent::CreateArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& dirToArchive) - { - bool success = false; - auto createArchiveCallback = [&success](bool result, AZStd::string consoleOutput) { - success = result; - }; - - AZStd::string commandLineArgs = Platform::GetCreateArchiveCommand(archivePath, dirToArchive); - - if (commandLineArgs.empty()) + if (!CheckParamsForCreate(archivePath, dirToArchive)) { - // The platform-specific implementation has already thrown its own error, no need to throw another one - return false; + std::promise p; + p.set_value(false); + return p.get_future(); } - LaunchZipExe(m_zipExePath, commandLineArgs, createArchiveCallback, AZ::Uuid::CreateNull(), dirToArchive, false); - return success; - } + auto FnCreateArchive = [this, archivePath, dirToArchive](std::promise&& p) -> void + { + auto archive = m_archive->OpenArchive(archivePath, {}, AZ::IO::INestedArchive::FLAGS_CREATE_NEW); + if (!archive) + { + AZ_Error(s_traceName, false, "Failed to create archive file '%s'", archivePath.c_str()); + p.set_value(false); + return; + } - void ArchiveComponent::ExtractArchive(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseCallback& respCallback) - { - ArchiveResponseOutputCallback responseHandler = [respCallback](bool result, AZStd::string /*outputStr*/) { respCallback(result); }; - ExtractArchiveOutput(archivePath, destinationPath, taskHandle, responseHandler); - } + auto foundFiles = AzFramework::FileFunc::FindFilesInPath(dirToArchive, "*", true); + if (!foundFiles.IsSuccess()) + { + AZ_Error(s_traceName, false, "Failed to find file listing under directory '%d'", dirToArchive.c_str()); + p.set_value(false); + return; + } - void ArchiveComponent::ExtractArchiveOutput(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) - { - AZStd::string commandLineArgs = Platform::GetExtractArchiveCommand(archivePath, destinationPath, true); + bool success = true; + AZStd::vector fileBuffer; + const AZ::IO::Path workingPath{ dirToArchive }; - if (commandLineArgs.empty()) - { - // The platform-specific implementation has already thrown its own error, no need to throw another one - return; - } + for (const auto& fileName : foundFiles.GetValue()) + { + bool thisSuccess = false; - LaunchZipExe(m_unzipExePath, commandLineArgs, respCallback, taskHandle); - } + AZ::IO::PathView relativePath = AZ::IO::PathView{ fileName }.LexicallyRelative(workingPath); - void ArchiveComponent::ExtractArchiveWithoutRoot(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) - { - AZStd::string commandLineArgs = Platform::GetExtractArchiveCommand(archivePath, destinationPath, false); + AZ::IO::Path fullPath = (workingPath / relativePath); + if (ArchiveUtils::ReadFile(fullPath, AZ::IO::OpenMode::ModeRead, fileBuffer)) + { + int result = archive->UpdateFile( + relativePath.Native(), fileBuffer.data(), fileBuffer.size(), s_compressionMethod, + s_compressionLevel, s_compressionCodec); + + thisSuccess = (result == AZ::IO::ZipDir::ZD_ERROR_SUCCESS); + AZ_Error( + s_traceName, thisSuccess, "Error %d encountered while adding '%s' to archive '%.*s'", result, fileName.c_str(), + AZ_STRING_ARG(archive->GetFullPath().Native())); + } + else + { + AZ_Error( + s_traceName, false, "Error encountered while reading '%s' to add to archive '%.*s'", fileName.c_str(), + AZ_STRING_ARG(archive->GetFullPath().Native())); + } - if (commandLineArgs.empty()) - { - // The platform-specific implementation has already thrown its own error, no need to throw another one - return; - } + success = (success && thisSuccess); + } - LaunchZipExe(m_unzipExePath, commandLineArgs, respCallback, taskHandle); + archive.reset(); + p.set_value(success); + }; + + // Async task... + std::promise p; + std::future f = p.get_future(); + + AZStd::thread_desc threadDesc; + threadDesc.m_name = "Archive Task (Create)"; + m_threads.emplace_back(threadDesc, FnCreateArchive, AZStd::move(p)); + return f; } - void ArchiveComponent::ExtractFile(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) + + std::future ArchiveComponent::ExtractArchive( + const AZStd::string& archivePath, + const AZStd::string& destinationPath) { - AZStd::string commandLineArgs = AzToolsFramework::Platform::GetExtractFileCommand(archivePath, fileInArchive, destinationPath, overWrite); - if (commandLineArgs.empty()) + if (!CheckParamsForExtract(archivePath, destinationPath)) { - // The platform-specific implementation has already thrown its own error, no need to throw another one - return; + std::promise p; + p.set_value(false); + return p.get_future(); } - LaunchZipExe(m_unzipExePath, commandLineArgs, respCallback, taskHandle); - } - bool ArchiveComponent::ExtractFileBlocking(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite) - { - AZStd::string commandLineArgs = AzToolsFramework::Platform::GetExtractFileCommand(archivePath, fileInArchive, destinationPath, overWrite); - if (commandLineArgs.empty()) + auto FnExtractArchive = [this, archivePath, destinationPath](std::promise&& p) -> void { - // The platform-specific implementation has already thrown its own error, no need to throw another one - return false; - } + auto archive = m_archive->OpenArchive(archivePath, {}, AZ::IO::INestedArchive::FLAGS_READ_ONLY); + if (!archive) + { + AZ_Error(s_traceName, false, "Failed to open archive file '%s'", archivePath.c_str()); + p.set_value(false); + return; + } - bool success = false; - auto createArchiveCallback = [&success](bool result, AZStd::string consoleOutput) { - success = result; - }; - LaunchZipExe(m_unzipExePath, commandLineArgs, createArchiveCallback); - return success; - } + AZStd::vector filesInArchive; + if (int result = archive->ListAllFiles(filesInArchive); result != AZ::IO::ZipDir::ZD_ERROR_SUCCESS) + { + AZ_Error(s_traceName, false, "Failed to get list of files in archive '%s'", archivePath.c_str()); + p.set_value(false); + return; + } - void ArchiveComponent::ListFilesInArchive(const AZStd::string& archivePath, AZStd::vector& fileEntries, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) - { - AZStd::string commandLineArgs = Platform::GetListFilesInArchiveCommand(archivePath); - - auto parseOutput = [respCallback, &fileEntries](bool exitCode, AZStd::string consoleOutput) - { - Platform::ParseConsoleOutputFromListFilesInArchive(consoleOutput, fileEntries); - AZ::TickBus::QueueFunction(respCallback, exitCode, AZStd::move(consoleOutput)); - }; - LaunchZipExe(m_unzipExePath, commandLineArgs, parseOutput, taskHandle, "", true); - } + AZStd::vector fileBuffer; + AZ::IO::Path destination{ destinationPath }; + AZ::u64 fileSize = 0; + AZ::u64 numFilesWritten = 0; + AZ::u64 bytesWritten = 0; + AZ::IO::INestedArchive::Handle srcHandle{}; + AZ::IO::HandleType dstHandle = AZ::IO::InvalidHandle; + constexpr AZ::IO::OpenMode openMode = + (AZ::IO::OpenMode::ModeCreatePath | AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeUpdate); + + for (const auto& filePath : filesInArchive) + { + srcHandle = archive->FindFile(filePath.Native()); + AZ_Assert(srcHandle != nullptr, "File '%s' does not exist inside archive '%s'", filePath.c_str(), archivePath.c_str()); - bool ArchiveComponent::ListFilesInArchiveBlocking(const AZStd::string& archivePath, AZStd::vector& fileEntries) - { - AZStd::string listOutput; - AZStd::string commandLineArgs = Platform::GetListFilesInArchiveCommand(archivePath.c_str()); - bool success = false; + fileSize = (srcHandle != nullptr) ? archive->GetFileSize(srcHandle) : 0; + fileBuffer.resize_no_construct(fileSize); + if (auto result = archive->ReadFile(srcHandle, fileBuffer.data()); result != AZ::IO::ZipDir::ZD_ERROR_SUCCESS) + { + AZ_Error( + s_traceName, false, "Failed to read file '%s' in archive '%s' with error %d", filePath.c_str(), archivePath.c_str(), + result); + continue; + } - auto parseOutput = [&success, &fileEntries](bool result, AZStd::string consoleOutput) - { - Platform::ParseConsoleOutputFromListFilesInArchive(consoleOutput, fileEntries); - success = result; + AZ::IO::Path destinationFile = destination / filePath; + if (!m_fileIO->Open(destinationFile.c_str(), openMode, dstHandle)) + { + AZ_Error(s_traceName, false, "Failed to open '%s' for writing", destinationFile.c_str()); + continue; + } + + if (!m_fileIO->Write(dstHandle, fileBuffer.data(), fileSize, &bytesWritten)) + { + AZ_Error(s_traceName, false, "Failed to write destination file '%s'", destinationFile.c_str()); + } + else if (bytesWritten == fileSize) + { + ++numFilesWritten; + } + + m_fileIO->Close(dstHandle); + } + + p.set_value(numFilesWritten == filesInArchive.size()); }; - LaunchZipExe(m_unzipExePath, commandLineArgs, parseOutput, AZ::Uuid::CreateNull(), "", true); - return success; - } - void ArchiveComponent::AddFileToArchive(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& fileToAdd, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) - { - AZStd::string commandLineArgs = Platform::GetAddFileToArchiveCommand(archivePath, fileToAdd); - if (commandLineArgs.empty()) - { - // The platform-specific implementation has already thrown its own error, no need to throw another one - return; - } + // Async task... + std::promise p; + std::future f = p.get_future(); - LaunchZipExe(m_zipExePath, commandLineArgs, respCallback, taskHandle, workingDirectory); + AZStd::thread_desc threadDesc; + threadDesc.m_name = "Archive Task (Extract)"; + m_threads.emplace_back(threadDesc, FnExtractArchive, AZStd::move(p)); + return f; } - bool ArchiveComponent::AddFileToArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& fileToAdd) + + std::future ArchiveComponent::ExtractFile( + const AZStd::string& archivePath, + const AZStd::string& fileInArchive, + const AZStd::string& destinationPath) { - AZStd::string commandLineArgs = Platform::GetAddFileToArchiveCommand(archivePath, fileToAdd); - if (commandLineArgs.empty()) + if (!CheckParamsForExtract(archivePath, destinationPath)) { - // The platform-specific implementation has already thrown its own error, no need to throw another one - return false; + std::promise p; + p.set_value(false); + return p.get_future(); } - bool success = false; - auto addFileToArchiveCallback = [&success](bool result, AZStd::string consoleOutput) { - success = result; - }; - LaunchZipExe(m_zipExePath, commandLineArgs, addFileToArchiveCallback, AZ::Uuid::CreateNull(), workingDirectory); - return success; - } + auto FnExtractFile = [this, archivePath, fileInArchive, destinationPath](std::promise&& p) -> void + { + auto archive = m_archive->OpenArchive(archivePath, {}, AZ::IO::INestedArchive::FLAGS_READ_ONLY); + if (!archive) + { + AZ_Error(s_traceName, false, "Failed to open archive file '%s'", archivePath.c_str()); + p.set_value(false); + return; + } - bool ArchiveComponent::AddFilesToArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& listFilePath) - { - bool success = false; + AZ::IO::INestedArchive::Handle fileHandle = archive->FindFile(fileInArchive); + if (!fileHandle) + { + AZ_Error(s_traceName, false, "File '%s' does not exist inside archive '%s'", fileInArchive.c_str(), archivePath.c_str()); + p.set_value(false); + return; + } - auto addFileToArchiveCallback = [&success](bool result, AZStd::string consoleOutput) { - success = result; + AZ::u64 fileSize = archive->GetFileSize(fileHandle); + AZStd::vector fileBuffer; + fileBuffer.resize_no_construct(fileSize); + + if (auto result = archive->ReadFile(fileHandle, fileBuffer.data()); result != AZ::IO::ZipDir::ZD_ERROR_SUCCESS) + { + AZ_Error( + s_traceName, false, "Failed to read file '%s' in archive '%s' with error %d", fileInArchive.c_str(), + archivePath.c_str(), result); + p.set_value(false); + return; + } + + AZ::IO::HandleType destFileHandle = AZ::IO::InvalidHandle; + AZ::IO::Path destinationFile{ destinationPath }; + destinationFile /= fileInArchive; + AZ::IO::OpenMode openMode = (AZ::IO::OpenMode::ModeCreatePath | AZ::IO::OpenMode::ModeWrite | AZ::IO::OpenMode::ModeUpdate); + if (!m_fileIO->Open(destinationFile.c_str(), openMode, destFileHandle)) + { + AZ_Error(s_traceName, false, "Failed to open destination file '%s' for writing", destinationFile.c_str()); + p.set_value(false); + return; + } + + AZ::u64 bytesWritten = 0; + if (!m_fileIO->Write(destFileHandle, fileBuffer.data(), fileSize, &bytesWritten)) + { + AZ_Error(s_traceName, false, "Failed to write destination file '%s'", destinationFile.c_str()); + } + + m_fileIO->Close(destFileHandle); + p.set_value(bytesWritten == fileSize); }; - AZStd::string commandLineArgs = Platform::GetAddFilesToArchiveCommand(archivePath.c_str(), listFilePath.c_str()); + // Async task... + std::promise p; + std::future f = p.get_future(); + + AZStd::thread_desc threadDesc; + threadDesc.m_name = "Archive Task (Extract Single)"; + m_threads.emplace_back(threadDesc, FnExtractFile, AZStd::move(p)); + return f; + } + - if (commandLineArgs.empty()) + bool ArchiveComponent::ListFilesInArchive(const AZStd::string& archivePath, AZStd::vector& outFileEntries) + { + if (!m_fileIO || !m_archive) { - // The platform-specific implementation has already thrown its own error, no need to throw another one return false; } - LaunchZipExe(m_zipExePath, commandLineArgs, addFileToArchiveCallback, AZ::Uuid::CreateNull(), workingDirectory); - return success; - } - void ArchiveComponent::AddFilesToArchive(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& listFilePath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) - { - AZStd::string commandLineArgs = Platform::GetAddFilesToArchiveCommand(archivePath, listFilePath); - if (commandLineArgs.empty()) + if (!m_fileIO->Exists(archivePath.c_str())) + { + AZ_Error(s_traceName, false, "Archive '%s' does not exist!", archivePath.c_str()); + return false; + } + + auto archive = m_archive->OpenArchive(archivePath, {}, AZ::IO::INestedArchive::FLAGS_READ_ONLY); + if (!archive) { - // The platform-specific implementation has already thrown its own error, no need to throw another one - return; + AZ_Error(s_traceName, false, "Failed to open archive file '%s'", archivePath.c_str()); + return false; } - LaunchZipExe(m_zipExePath, commandLineArgs, respCallback, taskHandle, workingDirectory); + AZStd::vector fileEntries; + int result = archive->ListAllFiles(fileEntries); + outFileEntries.clear(); + for (const auto& path : fileEntries) + { + outFileEntries.emplace_back(path.String()); + } + return (result == AZ::IO::ZipDir::ZD_ERROR_SUCCESS); } - bool ArchiveComponent::ExtractArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& destinationPath, bool extractWithRootDirectory) + std::future ArchiveComponent::AddFileToArchive( + const AZStd::string& archivePath, + const AZStd::string& workingDirectory, + const AZStd::string& fileToAdd) { - AZStd::string commandLineArgs = Platform::GetExtractArchiveCommand(archivePath, destinationPath, extractWithRootDirectory); - - if (commandLineArgs.empty()) + if (!CheckParamsForAdd(workingDirectory, fileToAdd)) { - // The platform-specific implementation has already thrown its own error, no need to throw another one - return false; + std::promise p; + p.set_value(false); + return p.get_future(); } - bool success = false; - auto extractArchiveCallback = [&success](bool result, AZStd::string consoleOutput) { - success = result; + auto FnAddFileToArchive = [this, archivePath, workingDirectory, fileToAdd](std::promise&& p) -> void + { + auto archive = m_archive->OpenArchive(archivePath); + if (!archive) + { + AZ_Error(s_traceName, false, "Failed to open archive file '%s'", archivePath.c_str()); + p.set_value(false); + return; + } + + AZ::IO::Path workingPath{ workingDirectory }; + AZ::IO::Path fullPath = workingPath / fileToAdd; + AZ::IO::PathView relativePath = AZ::IO::PathView{ fullPath }.LexicallyRelative(workingPath); + + AZStd::vector fileBuffer; + bool success = false; + if (ArchiveUtils::ReadFile(fullPath, AZ::IO::OpenMode::ModeRead, fileBuffer)) + { + int result = archive->UpdateFile( + relativePath.Native(), fileBuffer.data(), fileBuffer.size(), s_compressionMethod, + s_compressionLevel, s_compressionCodec); + + success = (result == AZ::IO::ZipDir::ZD_ERROR_SUCCESS); + AZ_Error( + s_traceName, success, "Error %d encountered while adding '%s' to archive '%.*s'", result, fileToAdd.c_str(), + AZ_STRING_ARG(archive->GetFullPath().Native())); + } + else + { + AZ_Error( + s_traceName, false, "Error encountered while reading '%s' to add to archive '%.*s'", fileToAdd.c_str(), + AZ_STRING_ARG(archive->GetFullPath().Native())); + } + + archive.reset(); + p.set_value(success); }; - LaunchZipExe(m_unzipExePath, commandLineArgs, extractArchiveCallback); - return success; + // Async task... + std::promise p; + std::future f = p.get_future(); + + AZStd::thread_desc threadDesc; + threadDesc.m_name = "Archive Task (Add Single)"; + m_threads.emplace_back(threadDesc, FnAddFileToArchive, AZStd::move(p)); + return f; } - void ArchiveComponent::CancelTasks(AZ::Uuid taskHandle) - { - AZStd::unique_lock lock(m_threadControlMutex); - auto it = m_threadInfoMap.find(taskHandle); - if (it == m_threadInfoMap.end()) + std::future ArchiveComponent::AddFilesToArchive( + const AZStd::string& archivePath, + const AZStd::string& workingDirectory, + const AZStd::string& listFilePath) + { + if (!CheckParamsForAdd(workingDirectory, listFilePath)) { - return; + std::promise p; + p.set_value(false); + return p.get_future(); } - ThreadInfo& info = it->second; - info.shouldStop = true; - m_cv.wait(lock, [&info]() { - return info.threads.size() == 0; - }); - m_threadInfoMap.erase(it); - } - - void ArchiveComponent::LaunchZipExe(const AZStd::string& exePath, const AZStd::string& commandLineArgs, const ArchiveResponseOutputCallback& respCallback, AZ::Uuid taskHandle, const AZStd::string& workingDir, bool captureOutput) - { - auto sevenZJob = [=]() + auto FnAddFilesToArchive = [this, archivePath, workingDirectory, listFilePath](std::promise&& p) -> void { - if (!taskHandle.IsNull()) + auto archive = m_archive->OpenArchive(archivePath); + if (!archive) { - AZStd::unique_lock lock(m_threadControlMutex); - m_threadInfoMap[taskHandle].threads.insert(AZStd::this_thread::get_id()); - m_cv.notify_all(); + AZ_Error(s_traceName, false, "Failed to open archive file '%s'", archivePath.c_str()); + p.set_value(false); + return; } - AzFramework::ProcessLauncher::ProcessLaunchInfo info; - info.m_commandlineParameters = exePath + " " + commandLineArgs; - - info.m_showWindow = false; - if (!workingDir.empty()) - { - info.m_workingDirectory = workingDir; - } - AZStd::unique_ptr watcher(AzFramework::ProcessWatcher::LaunchProcess(info, AzFramework::ProcessCommunicationType::COMMUNICATOR_TYPE_STDINOUT)); + bool success = true; // starts true and turns false when any error is encountered. + AZ::IO::Path basePath{ workingDirectory }; - AZStd::string consoleOutput; - AZ::u32 exitCode = static_cast(SevenZipExitCode::UserStoppedProcess); - if (watcher) + auto PerLineCallback = [&success, &basePath, &archive](AZStd::string_view filePathLine) -> void { - // callback requires output captured from 7z - if (captureOutput) + AZStd::vector fileBuffer; + AZ::IO::Path fullPath = (basePath / filePathLine); + if (ArchiveUtils::ReadFile(fullPath, AZ::IO::OpenMode::ModeRead, fileBuffer)) { - AZStd::string consoleBuffer; - while (watcher->IsProcessRunning(&exitCode)) - { - if (!taskHandle.IsNull()) - { - AZStd::unique_lock lock(m_threadControlMutex); - if (m_threadInfoMap[taskHandle].shouldStop) - { - watcher->TerminateProcess(static_cast(SevenZipExitCode::UserStoppedProcess)); - } - } - watcher->WaitForProcessToExit(g_sleepDuration, &exitCode); - AZ::u32 outputSize = watcher->GetCommunicator()->PeekOutput(); - if (outputSize) - { - consoleBuffer.resize(outputSize); - watcher->GetCommunicator()->ReadOutput(consoleBuffer.data(), outputSize); - consoleOutput += consoleBuffer; - } - } + int result = archive->UpdateFile( + filePathLine, fileBuffer.data(), fileBuffer.size(), s_compressionMethod, + s_compressionLevel, s_compressionCodec); + + bool thisSuccess = (result == AZ::IO::ZipDir::ZD_ERROR_SUCCESS); + success = (success && thisSuccess); + AZ_Error( + s_traceName, thisSuccess, "Error %d encountered while adding '%.*s' to archive '%.*s'", result, + AZ_STRING_ARG(filePathLine), AZ_STRING_ARG(archive->GetFullPath().Native())); } else { - ConsoleEchoCommunicator echoCommunicator(watcher->GetCommunicator()); - while (watcher->IsProcessRunning(&exitCode)) - { - if (!taskHandle.IsNull()) - { - AZStd::unique_lock lock(m_threadControlMutex); - if (m_threadInfoMap[taskHandle].shouldStop) - { - watcher->TerminateProcess(static_cast(SevenZipExitCode::UserStoppedProcess)); - } - } - watcher->WaitForProcessToExit(g_sleepDuration, &exitCode); - echoCommunicator.Pump(); - } + AZ_Error( + s_traceName, false, "Error encountered while reading '%.*s' to add to archive '%.*s'", AZ_STRING_ARG(filePathLine), + AZ_STRING_ARG(archive->GetFullPath().Native())); } - } + }; - if (taskHandle.IsNull()) - { - respCallback(exitCode == static_cast(SevenZipExitCode::NoError), AZStd::move(consoleOutput)); - } - else + ArchiveUtils::ProcessFileList(listFilePath, PerLineCallback); + + archive.reset(); + p.set_value(success); + }; + + // Async task... + std::promise p; + std::future f = p.get_future(); + + AZStd::thread_desc threadDesc; + threadDesc.m_name = "Archive Task (Add)"; + m_threads.emplace_back(threadDesc, FnAddFilesToArchive, AZStd::move(p)); + return f; + } + + + bool ArchiveComponent::CheckParamsForAdd(const AZStd::string& directory, const AZStd::string& file) + { + if (!m_fileIO || !m_archive) + { + return false; + } + + if (!m_fileIO->IsDirectory(directory.c_str())) + { + AZ_Error( + s_traceName, false, "Working directory '%s' is not a directory or doesn't exist!", directory.c_str()); + return false; + } + + if (!file.empty()) + { + auto filePath = AZ::IO::Path{ directory } / file; + if (!m_fileIO->Exists(filePath.c_str()) || m_fileIO->IsDirectory(filePath.c_str())) { - AZ::TickBus::QueueFunction(respCallback, (exitCode == static_cast(SevenZipExitCode::NoError)), AZStd::move(consoleOutput)); + AZ_Error(s_traceName, false, "File list '%s' is a directory or doesn't exist!", filePath.c_str()); + return false; } + } + + return true; + } - if (!taskHandle.IsNull()) + bool ArchiveComponent::CheckParamsForExtract(const AZStd::string& archive, const AZStd::string& directory) + { + if (!m_fileIO || !m_archive) + { + return false; + } + + if (!m_fileIO->Exists(archive.c_str())) + { + AZ_Error(s_traceName, false, "Archive '%s' does not exist!", archive.c_str()); + return false; + } + + if (!m_fileIO->Exists(directory.c_str())) + { + if (!m_fileIO->CreatePath(directory.c_str())) { - AZStd::unique_lock lock(m_threadControlMutex); - ThreadInfo& tInfo = m_threadInfoMap[taskHandle]; - tInfo.threads.erase(AZStd::this_thread::get_id()); - m_cv.notify_all(); + AZ_Error(s_traceName, false, "Failed to create destination directory '%s'", directory.c_str()); + return false; } - }; - if (!taskHandle.IsNull()) + } + + return true; + } + + bool ArchiveComponent::CheckParamsForCreate(const AZStd::string& archive, const AZStd::string& directory) + { + if (!m_fileIO || !m_archive) + { + return false; + } + + if (m_fileIO->Exists(archive.c_str())) { - AZStd::thread processThread(sevenZJob); - AZStd::unique_lock lock(m_threadControlMutex); - ThreadInfo& info = m_threadInfoMap[taskHandle]; - m_cv.wait(lock, [&info, &processThread]() { - return info.threads.find(processThread.get_id()) != info.threads.end(); - }); - processThread.detach(); + AZ_Error(s_traceName, false, "Archive file '%s' already exists, cannot create a new archive there!"); + return false; } - else + + if (!m_fileIO->IsDirectory(directory.c_str())) { - sevenZJob(); + AZ_Error(s_traceName, false, "Directory '%s' is not a directory or doesn't exist!", directory.c_str()); + return false; } + + return true; } + } // namespace AzToolsFramework diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveComponent.h b/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveComponent.h index fc344fc5a6..e2b437bcf2 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveComponent.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/ArchiveComponent.h @@ -10,80 +10,76 @@ #include #include +#include #include #include #include #include +#include #include namespace AzToolsFramework { - enum class SevenZipExitCode : AZ::u32 - { - NoError = 0, - Warning = 1, - FatalError = 2, - CommandLineError = 7, - NotEnoughMemory = 8, - UserStoppedProcess = 255 - }; - - // the ArchiveComponent's job is to execute zip commands. - // it parses the status of zip commands and returns results. + // the ArchiveComponent's job is to create and manipulate zip archives. class ArchiveComponent : public AZ::Component - , private ArchiveCommands::Bus::Handler + , private ArchiveCommandsBus::Handler { public: - AZ_COMPONENT(ArchiveComponent, "{A19EEA33-3736-447F-ACF7-DAA4B6A179AA}") + AZ_COMPONENT(ArchiveComponent, "{A19EEA33-3736-447F-ACF7-DAA4B6A179AA}"); ArchiveComponent() = default; ~ArchiveComponent() override = default; + ArchiveComponent(const ArchiveComponent&) = delete; + ArchiveComponent& operator=(const ArchiveComponent&) = delete; + ////////////////////////////////////////////////////////////////////////// // AZ::Component overrides void Activate() override; void Deactivate() override; ////////////////////////////////////////////////////////////////////////// - private: + + protected: static void Reflect(AZ::ReflectContext* context); ////////////////////////////////////////////////////////////////////////// - // ArchiveCommands::Bus::Handler overrides - void CreateArchive(const AZStd::string& archivePath, const AZStd::string& dirToArchive, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - bool CreateArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& dirToArchive) override; - bool ExtractArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& destinationPath, bool extractWithRootDirectory) override; - void ExtractArchive(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseCallback& respCallback) override; - void ExtractArchiveOutput(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - void ExtractArchiveWithoutRoot(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - void ExtractFile(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - bool ExtractFileBlocking(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite) override; - void ListFilesInArchive(const AZStd::string& archivePath, AZStd::vector& fileEntries, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - bool ListFilesInArchiveBlocking(const AZStd::string& archivePath, AZStd::vector& fileEntries) override; - void AddFileToArchive(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& fileToAdd, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - bool AddFileToArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& fileToAdd) override; - bool AddFilesToArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& listFilePath) override; - void AddFilesToArchive(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& listFilePath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - void CancelTasks(AZ::Uuid taskHandle) override; + // ArchiveCommandsBus::Handler overrides + [[nodiscard]] std::future CreateArchive( + const AZStd::string& archivePath, + const AZStd::string& dirToArchive) override; + + [[nodiscard]] std::future ExtractArchive( + const AZStd::string& archivePath, + const AZStd::string& destinationPath) override; + + [[nodiscard]] std::future ExtractFile( + const AZStd::string& archivePath, + const AZStd::string& fileInArchive, + const AZStd::string& destinationPath) override; + + bool ListFilesInArchive(const AZStd::string& archivePath, AZStd::vector& outFileEntries) override; + + [[nodiscard]] std::future AddFileToArchive( + const AZStd::string& archivePath, + const AZStd::string& workingDirectory, + const AZStd::string& fileToAdd) override; + + [[nodiscard]] std::future AddFilesToArchive( + const AZStd::string& archivePath, + const AZStd::string& workingDirectory, + const AZStd::string& listFilePath) override; ////////////////////////////////////////////////////////////////////////// - - // Launches the input zip exe as a background child process in a detached background thread, if the task handle is not null - // otherwise launches input zip exe in the calling thread. - void LaunchZipExe(const AZStd::string& exePath, const AZStd::string& commandLineArgs, const ArchiveResponseOutputCallback& respCallback, AZ::Uuid taskHandle = AZ::Uuid::CreateNull(), const AZStd::string& workingDir = "", bool captureOutput = false); - - AZStd::string m_zipExePath; - AZStd::string m_unzipExePath; - - // Struct for tracking background threads/tasks - struct ThreadInfo - { - bool shouldStop = false; - AZStd::set threads; - }; - - AZStd::mutex m_threadControlMutex; // Guards m_threadInfoMap - AZStd::condition_variable m_cv; - AZStd::unordered_map m_threadInfoMap; + + private: + AZ::IO::FileIOBase* m_fileIO = nullptr; + AZ::IO::IArchive* m_archive = nullptr; + AZStd::vector m_threads; + + bool CheckParamsForAdd(const AZStd::string& directory, const AZStd::string& file); + bool CheckParamsForExtract(const AZStd::string& archive, const AZStd::string& directory); + bool CheckParamsForCreate(const AZStd::string& archive, const AZStd::string& directory); }; + } // namespace AzToolsFramework diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/NullArchiveComponent.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/NullArchiveComponent.cpp index f82724a721..972e9d588f 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/NullArchiveComponent.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/NullArchiveComponent.cpp @@ -16,91 +16,62 @@ namespace AzToolsFramework void NullArchiveComponent::Activate() { - ArchiveCommands::Bus::Handler::BusConnect(); + ArchiveCommandsBus::Handler::BusConnect(); } void NullArchiveComponent::Deactivate() { - ArchiveCommands::Bus::Handler::BusDisconnect(); + ArchiveCommandsBus::Handler::BusDisconnect(); } - bool NullArchiveComponent::ExtractArchiveBlocking(const AZStd::string& /*archivePath*/, const AZStd::string& /*destinationPath*/, bool /*extractWithRootDirectory*/) + std::future DefaultFuture() { - return false; - } - - void NullArchiveComponent::ExtractArchive(const AZStd::string& /*archivePath*/, const AZStd::string& /*destinationPath*/, AZ::Uuid /*taskHandle*/, const ArchiveResponseCallback& respCallback) - { - AZ::TickBus::QueueFunction(respCallback, false); + std::promise p; + p.set_value(false); + return p.get_future(); } - void NullArchiveComponent::ExtractArchiveOutput(const AZStd::string& /*archivePath*/, const AZStd::string& /*destinationPath*/, AZ::Uuid /*taskHandle*/, const ArchiveResponseOutputCallback& respCallback) + std::future NullArchiveComponent::CreateArchive( + const AZStd::string& /*archivePath*/, + const AZStd::string& /*dirToArchive*/) { - AZ::TickBus::QueueFunction(respCallback, false, AZStd::string()); + return DefaultFuture(); } - void NullArchiveComponent::ExtractArchiveWithoutRoot(const AZStd::string& /*archivePath*/, const AZStd::string& /*destinationPath*/, AZ::Uuid /*taskHandle*/, const ArchiveResponseOutputCallback& respCallback) + std::future NullArchiveComponent::ExtractArchive( + const AZStd::string& /*archivePath*/, + const AZStd::string& /*destinationPath*/) { - AZ::TickBus::QueueFunction(respCallback, false, AZStd::string()); + return DefaultFuture(); } - void NullArchiveComponent::ExtractFile(const AZStd::string& /*archivePath*/, const AZStd::string& /*fileInArchive*/, const AZStd::string& /*destinationPath*/, bool /*overWrite*/, AZ::Uuid /*taskHandle*/, const ArchiveResponseOutputCallback& respCallback) + std::future NullArchiveComponent::ExtractFile( + const AZStd::string& /*archivePath*/, + const AZStd::string& /*fileInArchive*/, + const AZStd::string& /*destinationPath*/) { - // Always report we failed to extract - AZ::TickBus::QueueFunction(respCallback, false, AZStd::string()); + return DefaultFuture(); } - bool NullArchiveComponent::ExtractFileBlocking(const AZStd::string& /*archivePath*/, const AZStd::string& /*fileInArchive*/, const AZStd::string& /*destinationPath*/, bool /*overWrite*/) + bool NullArchiveComponent::ListFilesInArchive(const AZStd::string& /*archivePath*/, AZStd::vector& /*outFileEntries*/) { return false; } - void NullArchiveComponent::ListFilesInArchive(const AZStd::string& /*archivePath*/, AZStd::vector& /*consoleOutput*/, AZ::Uuid /*taskHandle*/, const ArchiveResponseOutputCallback& respCallback) + std::future NullArchiveComponent::AddFileToArchive( + const AZStd::string& /*archivePath*/, + const AZStd::string& /*fileToAdd*/, + const AZStd::string& /*pathInArchive*/) { - // Always report we failed to extract - AZ::TickBus::QueueFunction(respCallback, false, AZStd::string()); - } - - bool NullArchiveComponent::ListFilesInArchiveBlocking(const AZStd::string& /*archivePath*/, AZStd::vector& /*consoleOutput*/) - { - return false; - } - - void NullArchiveComponent::AddFileToArchive(const AZStd::string& /*archivePath*/, const AZStd::string& /*fileToAdd*/, const AZStd::string& /*pathInArchive*/, AZ::Uuid /*taskHandle*/, const ArchiveResponseOutputCallback& respCallback) - { - // Always report we failed to extract - AZ::TickBus::QueueFunction(respCallback, false, AZStd::string()); - } - - bool NullArchiveComponent::AddFileToArchiveBlocking(const AZStd::string& /*archivePath*/, const AZStd::string& /*fileToAdd*/, const AZStd::string& /*pathInArchive*/) - { - return false; - } - - bool NullArchiveComponent::AddFilesToArchiveBlocking(const AZStd::string& /*archivePath*/, const AZStd::string& /*workingDirectory*/, const AZStd::string& /*listFilePath*/) - { - return false; - } - - void NullArchiveComponent::AddFilesToArchive(const AZStd::string& /*archivePath*/, const AZStd::string& /*workingDirectory*/, const AZStd::string& /*listFilePath*/, AZ::Uuid /*taskHandle*/, const ArchiveResponseOutputCallback& respCallback) - { - // Always report we failed to extract - AZ::TickBus::QueueFunction(respCallback, false, AZStd::string()); - } - - void NullArchiveComponent::CreateArchive(const AZStd::string& /*archivePath*/, const AZStd::string& /*dirToArchive*/, AZ::Uuid /*taskHandle*/, const ArchiveResponseOutputCallback& respCallback) - { - // Always report we failed to extract - AZ::TickBus::QueueFunction(respCallback, false, AZStd::string()); - } - - bool NullArchiveComponent::CreateArchiveBlocking(const AZStd::string& /*archivePath*/, const AZStd::string& /*dirToArchive*/) - { - return false; + return DefaultFuture(); } - void NullArchiveComponent::CancelTasks(AZ::Uuid /*taskHandle*/) + std::future NullArchiveComponent::AddFilesToArchive( + const AZStd::string& /*archivePath*/, + const AZStd::string& /*workingDirectory*/, + const AZStd::string& /*listFilePath*/) { + return DefaultFuture(); } void NullArchiveComponent::Reflect(AZ::ReflectContext* context) diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/NullArchiveComponent.h b/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/NullArchiveComponent.h index 2023490351..9ce33d0c85 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/NullArchiveComponent.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/Archive/NullArchiveComponent.h @@ -15,7 +15,7 @@ namespace AzToolsFramework { class NullArchiveComponent : public AZ::Component - , private ArchiveCommands::Bus::Handler + , private ArchiveCommandsBus::Handler { public: AZ_COMPONENT(NullArchiveComponent, "{D665B6B1-5FF4-4203-B19F-BBDB82587129}") @@ -32,23 +32,31 @@ namespace AzToolsFramework static void Reflect(AZ::ReflectContext* context); ////////////////////////////////////////////////////////////////////////// - // ArchiveCommands::Bus::Handler overrides - // ArchiveCommands::Bus::Handler overrides - void CreateArchive(const AZStd::string& archivePath, const AZStd::string& dirToArchive, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - bool CreateArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& dirToArchive) override; - bool ExtractArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& destinationPath, bool extractWithRootDirectory) override; - void ExtractArchive(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseCallback& respCallback) override; - void ExtractArchiveOutput(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - void ExtractArchiveWithoutRoot(const AZStd::string& archivePath, const AZStd::string& destinationPath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - void ExtractFile(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - bool ExtractFileBlocking(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite) override; - void ListFilesInArchive(const AZStd::string& archivePath, AZStd::vector& consoleOutput, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - bool ListFilesInArchiveBlocking(const AZStd::string& archivePath, AZStd::vector& consoleOutput) override; - void AddFileToArchive(const AZStd::string& archivePath, const AZStd::string& fileToAdd, const AZStd::string& pathInArchive, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - bool AddFileToArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& fileToAdd, const AZStd::string& pathInArchive) override; - bool AddFilesToArchiveBlocking(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& listFilePath) override; - void AddFilesToArchive(const AZStd::string& archivePath, const AZStd::string& workingDirectory, const AZStd::string& listFilePath, AZ::Uuid taskHandle, const ArchiveResponseOutputCallback& respCallback) override; - void CancelTasks(AZ::Uuid taskHandle) override; + // ArchiveCommandsBus::Handler overrides + [[nodiscard]] std::future CreateArchive( + const AZStd::string& archivePath, + const AZStd::string& dirToArchive) override; + + [[nodiscard]] std::future ExtractArchive( + const AZStd::string& archivePath, + const AZStd::string& destinationPath) override; + + [[nodiscard]] std::future ExtractFile( + const AZStd::string& archivePath, + const AZStd::string& fileInArchive, + const AZStd::string& destinationPath) override; + + bool ListFilesInArchive(const AZStd::string& archivePath, AZStd::vector& outFileEntries) override; + + [[nodiscard]] std::future AddFileToArchive( + const AZStd::string& archivePath, + const AZStd::string& workingDirectory, + const AZStd::string& fileToAdd) override; + + [[nodiscard]] std::future AddFilesToArchive( + const AZStd::string& archivePath, + const AZStd::string& workingDirectory, + const AZStd::string& listFilePath) override; ////////////////////////////////////////////////////////////////////////// }; } // namespace AzToolsFramework diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBundle/AssetBundleComponent.cpp b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBundle/AssetBundleComponent.cpp index 7083737cf7..8924907e81 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBundle/AssetBundleComponent.cpp +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBundle/AssetBundleComponent.cpp @@ -32,12 +32,10 @@ namespace AzToolsFramework const int NumOfBytesInMB = 1024 * 1024; const int ManifestFileSizeBufferInBytes = 10 * 1024; // 10 KB const float AssetCatalogFileSizeBufferPercentage = 1.0f; - using ArchiveCommandsBus = AzToolsFramework::ArchiveCommands::Bus; using AssetCatalogRequestBus = AZ::Data::AssetCatalogRequestBus; const char AssetBundleComponent::DeltaCatalogName[] = "DeltaCatalog.xml"; - constexpr int SleepTimeMS = 250; constexpr int InjectFileRetryCount = 4; @@ -136,7 +134,7 @@ namespace AzToolsFramework AZ_TracePrintf(logWindowName, "Gathering file entries in source pak file \"%s\".\n", sourcePak.c_str()); bool result = false; AZStd::vector fileEntries; - ArchiveCommandsBus::BroadcastResult(result, &AzToolsFramework::ArchiveCommands::ListFilesInArchiveBlocking, normalizedSourcePakPath, fileEntries); + ArchiveCommandsBus::BroadcastResult(result, &AzToolsFramework::ArchiveCommandsBus::Events::ListFilesInArchive, normalizedSourcePakPath, fileEntries); // This ebus currently always returns false as the result, as it is believed that the 7z process is // being terminated by the user instead of ending gracefully. Check against an empty fileList instead // as a result. @@ -606,15 +604,17 @@ namespace AzToolsFramework { AZ_TracePrintf(logWindowName, "Injecting file (%s) into bundle (%s).\n", filePath.c_str(), archiveFilePath.c_str()); bool fileAddedToArchive = false; + std::future fileAdded; int retryCount = InjectFileRetryCount; + while (!fileAddedToArchive && retryCount) { - ArchiveCommandsBus::BroadcastResult(fileAddedToArchive, &AzToolsFramework::ArchiveCommands::AddFileToArchiveBlocking, archiveFilePath, workingDirectory, filePath); + ArchiveCommandsBus::BroadcastResult(fileAdded, &AzToolsFramework::ArchiveCommandsBus::Events::AddFileToArchive, archiveFilePath, workingDirectory, filePath); --retryCount; + fileAddedToArchive = fileAdded.get(); if (!fileAddedToArchive && retryCount) { AZ_Error(logWindowName, false, "Failed to insert file (%s) into bundle (%s). Retrying.", filePath.c_str(), archiveFilePath.c_str()); - AZStd::this_thread::sleep_for(AZStd::chrono::milliseconds(SleepTimeMS)); } } @@ -627,7 +627,11 @@ namespace AzToolsFramework bool AssetBundleComponent::InjectFile(const AZStd::string& filePath, const AZStd::string& sourcePak) { - return InjectFile(filePath, sourcePak, ""); + // When no working directory is specified, assume that the file being injected goes into the root of the archive. + // The filePath should be an absolute path, making the workingDirectory be the path leading up to the file. + AZ::IO::PathView fullFilePath{ filePath, AZ::IO::PosixPathSeparator }; + AZ::IO::Path workingDir{ fullFilePath.ParentPath() }; + return InjectFile(filePath, sourcePak, workingDir.c_str()); } bool AssetBundleComponent::InjectFiles(const AZStd::vector& fileEntries, const AZStd::string& sourcePak, const char* workingDirectory) @@ -668,8 +672,9 @@ namespace AzToolsFramework } } - bool filesAddedToArchive = false; - AzToolsFramework::ArchiveCommandsBus::BroadcastResult(filesAddedToArchive, &AzToolsFramework::ArchiveCommands::AddFilesToArchiveBlocking, sourcePak, workingDirectory, listFilePath); + std::future filesAdded; + AzToolsFramework::ArchiveCommandsBus::BroadcastResult(filesAdded, &AzToolsFramework::ArchiveCommands::AddFilesToArchive, sourcePak, workingDirectory, listFilePath); + bool filesAddedToArchive = filesAdded.get(); if (!filesAddedToArchive) { AZ_Error(logWindowName, false, "Failed to insert files into bundle (%s).\n", sourcePak.c_str()); @@ -688,7 +693,6 @@ namespace AzToolsFramework { // open the manifest and deserialize it bool manifestExtracted = false; - const bool overwriteExisting = true; TemporaryDir tempDir(sourcePak); if (!tempDir.m_result) @@ -698,7 +702,10 @@ namespace AzToolsFramework AZStd::string manifestFilePath; AzFramework::StringFunc::Path::ConstructFull(tempDir.m_tempFolderPath.c_str(), AzFramework::AssetBundleManifest::s_manifestFileName, manifestFilePath, true); - ArchiveCommandsBus::BroadcastResult(manifestExtracted, &ArchiveCommandsBus::Events::ExtractFileBlocking, sourcePak, AzFramework::AssetBundleManifest::s_manifestFileName, tempDir.m_tempFolderPath, overwriteExisting); + + std::future extractResult; + ArchiveCommandsBus::BroadcastResult(extractResult, &ArchiveCommandsBus::Events::ExtractFile, sourcePak, AzFramework::AssetBundleManifest::s_manifestFileName, tempDir.m_tempFolderPath); + manifestExtracted = extractResult.get(); if (!manifestExtracted) { AZ_Error(logWindowName, false, "Failed to extract existing manifest from archive \"%s\".", sourcePak.c_str()); diff --git a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBundle/AssetBundleComponent.h b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBundle/AssetBundleComponent.h index 307777cc1b..17a9dd40f5 100644 --- a/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBundle/AssetBundleComponent.h +++ b/Code/Framework/AzToolsFramework/AzToolsFramework/AssetBundle/AssetBundleComponent.h @@ -57,7 +57,7 @@ namespace AzToolsFramework //! Returns true if the file at filePath was successfully injected into the bundle at sourcePak static bool InjectFile(const AZStd::string& filePath, const AZStd::string& sourcePak, const char* workingDirectory); - //! Inject the files with relative filePaths which espect to the working directory into the bundle at sourcePak + //! Inject the files with relative filePaths with respect to the working directory into the bundle at sourcePak //! Returns true if the file at filePath was successfully injected into the bundle at sourcePak static bool InjectFiles(const AZStd::vector& fileEntries, const AZStd::string& sourcePak, const char* workingDirectory); diff --git a/Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework/Archive/ArchiveComponent_Linux.cpp b/Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework/Archive/ArchiveComponent_Linux.cpp deleted file mode 100644 index cc8b9492e8..0000000000 --- a/Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework/Archive/ArchiveComponent_Linux.cpp +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (c) Contributors to the Open 3D Engine Project. - * For complete copyright and license terms please see the LICENSE at the root of this distribution. - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - * - */ - -#include -#include -#include -#include - -namespace AzToolsFramework -{ - namespace Platform - { - [[maybe_unused]] static const char ErrorChannel[] = "ArchiveComponent_Linux"; - - static const char ZipExePath[] = R"(/usr/bin/zip)"; - static const char UnzipExePath[] = R"(/usr/bin/unzip)"; - - static const char CreateArchiveCmd[] = "-r \"%s\" . -i *"; - - static const char ExtractArchiveCmd[] = R"(-o "%s" -d "%s")"; - - static const char AddFileCmd[] = R"("%s" "%s")"; - - static const char ExtractFileCmd[] = R"(%s "%s" %s)"; - static const char ExtractFileDestination[] = R"(%s "%s" "%s" -d "%s")"; - static const char ExtractOverwrite[] = "-o"; - static const char ExtractSkipExisting[] = "-n"; - static const char ListFilesInArchiveCmd[] = "-l %s"; - - AZStd::string GetZipExePath() - { - return ZipExePath; - } - - AZStd::string GetUnzipExePath() - { - return UnzipExePath; - } - - AZ::Outcome MakePath(const AZStd::string& path) - { - // Create the folder if it does not already exist - if (!AZ::IO::FileIOBase::GetInstance()->Exists(path.c_str())) - { - auto result = AZ::IO::FileIOBase::GetInstance()->CreatePath(path.c_str()); - if (!result) - { - return AZ::Failure(AZStd::string::format("Path creation failed. Input path: %s \n", path.c_str())); - } - } - - return AZ::Success(path); - } - - AZ::Outcome MakeCreateArchivePath(const AZStd::string& archivePath) - { - // Remove the file name from the input path - // /some/folder/path/archive.zip -> /some/folder/path/ - AZStd::string strippedArchivePath = archivePath; - AzFramework::StringFunc::Path::StripFullName(strippedArchivePath); - - if (strippedArchivePath.empty()) - { - return AZ::Failure(AZStd::string::format("Stripped path name is empty. Cancelling path creation. Input path: %s\n", archivePath.c_str())); - } - - return MakePath(strippedArchivePath); - } - - AZ::Outcome MakeExtractArchivePath(const AZStd::string& archivePath, const AZStd::string& destinationPath, bool includeRoot) - { - if(!includeRoot) - { - // Create the folder for the input destination path with no modifications - // /path/to/destination/ - return MakePath(destinationPath); - } - - // Get the name of the input archive. This will be the name of the root folder for the archive extraction - // /some/folder/path/archive.zip -> archive - AZStd::string zipFileName; - bool result = AzFramework::StringFunc::Path::GetFileName(archivePath.c_str(), zipFileName); - if(!result) - { - return AZ::Failure(AZStd::string::format("Failed to get name of zip file from the archive path. Cancelling path creation. \n Input Archive Path: %s \n", archivePath.c_str())); - } - - // Append the root folder name to the end of the destination path - // /path/to/destination/ + archive -> /path/to/destination/archive - AZStd::string destinationPathWithRoot; - result = AzFramework::StringFunc::Path::Join(destinationPath.c_str(), zipFileName.c_str(), destinationPathWithRoot); - if(!result) - { - return AZ::Failure(AZStd::string::format("Failed to append zip file name to the destination path. Cancelling path creation. \n Destination Path: %s \n Zip file name: %s \n", destinationPath.c_str(), zipFileName.c_str())); - } - - // Append a separator so that it is formatted like a folder - // /path/to/destination/archive -> /path/to/destination/archive/ - AzFramework::StringFunc::Path::AppendSeparator(destinationPathWithRoot); - return MakePath(destinationPathWithRoot); - } - - AZStd::string GetCreateArchiveCommand(const AZStd::string& archivePath, const AZStd::string& dirToArchive) - { - auto pathCreationResult = MakeCreateArchivePath(archivePath); - if (!pathCreationResult) - { - AZ_Error(ErrorChannel, false, "%s", pathCreationResult.GetError().c_str()); - return ""; - } - AZ_UNUSED(dirToArchive); - return AZStd::string::format(CreateArchiveCmd, archivePath.c_str()); - } - - AZStd::string GetExtractArchiveCommand(const AZStd::string& archivePath, const AZStd::string& destinationPath, bool includeRoot) - { - auto pathCreationResult = MakeExtractArchivePath(archivePath, destinationPath, includeRoot); - if (!pathCreationResult) - { - AZ_Error(ErrorChannel, false, "%s", pathCreationResult.GetError().c_str()); - return ""; - } - - return AZStd::string::format(ExtractArchiveCmd, archivePath.c_str(), pathCreationResult.GetValue().c_str()); - } - - AZStd::string GetAddFilesToArchiveCommand(const AZStd::string& /*archivePath*/, const AZStd::string& /*listFilePath*/) - { - // Adding files into a archive using a list file is not currently supported - return {}; - } - - bool IsAddFilesToArchiveCommandSupported() - { - // Adding files into a archive using a list file is not currently supported - return false; - } - - AZStd::string GetAddFileToArchiveCommand(const AZStd::string& archivePath, const AZStd::string& file) - { - if (!MakeCreateArchivePath(archivePath).IsSuccess()) - { - AZ_Error(ErrorChannel, false, "Unable to make path for ( %s ).\n", archivePath.c_str()); - return {}; - } - return AZStd::string::format(AddFileCmd, archivePath.c_str(), file.c_str()); - } - - AZStd::string GetExtractFileCommand(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite) - { - AZStd::string commandLineArgs; - if (destinationPath.empty()) - { - // Extract file in archive from archive path to the current directory, overwriting a file of the same name that exists there. - commandLineArgs = AZStd::string::format(ExtractFileCmd, overWrite ? ExtractOverwrite : ExtractSkipExisting, archivePath.c_str(), fileInArchive.c_str()); - } - else - { - if (!MakePath(destinationPath).IsSuccess()) - { - AZ_Error(ErrorChannel, false, "Unable to make path ( %s ).\n", destinationPath.c_str()); - return {}; - } - // Extract file in archive from archive path to destinationPath, overwriting a file of the same name that exists there. - commandLineArgs = AZStd::string::format(ExtractFileDestination, overWrite ? ExtractOverwrite : ExtractSkipExisting, archivePath.c_str(), fileInArchive.c_str(), destinationPath.c_str()); - } - - return commandLineArgs; - } - - AZStd::string GetListFilesInArchiveCommand(const AZStd::string& archivePath) - { - AZStd::string commandLineArgs = AZStd::string::format(ListFilesInArchiveCmd, archivePath.c_str()); - return commandLineArgs; - } - - /* - Sample Console Output of the unzip list command - - Archive: /var/folders/1q/12nyzqc913qgm532y2c98mnm6w4_qv/T/ArchiveTests-ra8oMy/TestArchive.pak - Length Date Time Name - --------- ---------- ----- ---- - 0 10-14-2019 15:22 testfolder/ - 1 10-14-2019 15:22 testfolder/folderfile.txt - 1 10-14-2019 15:22 basicfile.txt - 1 10-14-2019 15:22 basicfile2.txt - 0 10-14-2019 15:22 testfolder2/ - 1 10-14-2019 15:22 testfolder2/sharedfolderfile2.txt - 1 10-14-2019 15:22 testfolder2/sharedfolderfile.txt - 0 10-14-2019 15:22 testfolder3/ - 0 10-14-2019 15:22 testfolder3/testfolder4/ - 1 10-14-2019 15:22 testfolder3/testfolder4/depthfile.bat - --------- ------- - 6 10 files - */ - - void ParseConsoleOutputFromListFilesInArchive(const AZStd::string& consoleOutput, AZStd::vector& fileEntries) - { - AZStd::vector fileEntryData; - AzFramework::StringFunc::Tokenize(consoleOutput.c_str(), fileEntryData, "\n"); - int startingLineIdx = 3; // first line that might contain the file name - for (size_t lineIdx = startingLineIdx; lineIdx < fileEntryData.size(); ++lineIdx) - { - AZStd::string& line = fileEntryData[lineIdx]; - AZStd::vector lineEntryData; - AzFramework::StringFunc::Tokenize(line.c_str(), lineEntryData, " "); - AZStd::string& fileName = lineEntryData.back(); - - if(fileName.back() == AZ_CORRECT_FILESYSTEM_SEPARATOR) - { - // if the filename ends with a separator - // than it indicates that this is a directory - continue; - } - - if(fileName.compare("-------") == 0) - { - return; - } - - fileEntries.emplace_back(fileName); - } - } - } // namespace Platform -} // namespace AzToolsFramework diff --git a/Code/Framework/AzToolsFramework/Platform/Linux/platform_linux_files.cmake b/Code/Framework/AzToolsFramework/Platform/Linux/platform_linux_files.cmake index 84d1cf808f..c2c5a11c4c 100644 --- a/Code/Framework/AzToolsFramework/Platform/Linux/platform_linux_files.cmake +++ b/Code/Framework/AzToolsFramework/Platform/Linux/platform_linux_files.cmake @@ -7,5 +7,4 @@ # set(FILES - AzToolsFramework/Archive/ArchiveComponent_Linux.cpp ) diff --git a/Code/Framework/AzToolsFramework/Platform/Mac/AzToolsFramework/Archive/ArchiveComponent_Mac.cpp b/Code/Framework/AzToolsFramework/Platform/Mac/AzToolsFramework/Archive/ArchiveComponent_Mac.cpp deleted file mode 100644 index 67b7b9cf66..0000000000 --- a/Code/Framework/AzToolsFramework/Platform/Mac/AzToolsFramework/Archive/ArchiveComponent_Mac.cpp +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Copyright (c) Contributors to the Open 3D Engine Project. - * For complete copyright and license terms please see the LICENSE at the root of this distribution. - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - * - */ - -#include -#include -#include -#include - -namespace AzToolsFramework -{ - namespace Platform - { - const char ErrorChannel[] = "ArchiveComponent_OSX"; - - const char ZipExePath[] = R"(/usr/bin/zip)"; - const char UnzipExePath[] = R"(/usr/bin/unzip)"; - - // v Requires investigation, the correct cmd should be R"(-r "%s" "%s/")" but tests fail - const char CreateArchiveCmd[] = R"(-r "%s" .)"; - - const char ExtractArchiveCmd[] = R"(-o "%s" -d "%s")"; - - const char AddFileCmd[] = R"("%s" "%s" -X)"; - const char AddFilesCmd[] = R"("%s" -X %s)"; - - const char ExtractFileCmd[] = R"(%s "%s" %s)"; - const char ExtractFileDestination[] = R"(%s "%s" "%s" -d "%s")"; - const char ExtractOverwrite[] = "-o"; - const char ExtractSkipExisting[] = "-n"; - const char ListFilesInArchiveCmd[] = "-l %s"; - - AZStd::string GetZipExePath() - { - return ZipExePath; - } - - AZStd::string GetUnzipExePath() - { - return UnzipExePath; - } - - AZ::Outcome MakePath(const AZStd::string& path) - { - // Create the folder if it does not already exist - if (!AZ::IO::FileIOBase::GetInstance()->Exists(path.c_str())) - { - auto result = AZ::IO::FileIOBase::GetInstance()->CreatePath(path.c_str()); - if (!result) - { - return AZ::Failure(AZStd::string::format("Path creation failed. Input path: %s \n", path.c_str())); - } - } - - return AZ::Success(path); - } - - AZ::Outcome MakeCreateArchivePath(const AZStd::string& archivePath) - { - // Remove the file name from the input path - // /some/folder/path/archive.zip -> /some/folder/path/ - AZStd::string strippedArchivePath = archivePath; - AzFramework::StringFunc::Path::StripFullName(strippedArchivePath); - - if (strippedArchivePath.empty()) - { - return AZ::Failure(AZStd::string::format("Stripped path name is empty. Cancelling path creation. Input path: %s\n", archivePath.c_str())); - } - - return MakePath(strippedArchivePath); - } - - AZ::Outcome MakeExtractArchivePath(const AZStd::string& archivePath, const AZStd::string& destinationPath, bool includeRoot) - { - if(!includeRoot) - { - // Create the folder for the input destination path with no modifications - // /path/to/destination/ - return MakePath(destinationPath); - } - - // Get the name of the input archive. This will be the name of the root folder for the archive extraction - // /some/folder/path/archive.zip -> archive - AZStd::string zipFileName; - bool result = AzFramework::StringFunc::Path::GetFileName(archivePath.c_str(), zipFileName); - if(!result) - { - return AZ::Failure(AZStd::string::format("Failed to get name of zip file from the archive path. Cancelling path creation. \n Input Archive Path: %s \n", archivePath.c_str())); - } - - // Append the root folder name to the end of the destination path - // /path/to/destination/ + archive -> /path/to/destination/archive - AZStd::string destinationPathWithRoot; - result = AzFramework::StringFunc::Path::Join(destinationPath.c_str(), zipFileName.c_str(), destinationPathWithRoot); - if(!result) - { - return AZ::Failure(AZStd::string::format("Failed to append zip file name to the destination path. Cancelling path creation. \n Destination Path: %s \n Zip file name: %s \n", destinationPath.c_str(), zipFileName.c_str())); - } - - // Append a separator so that it is formatted like a folder - // /path/to/destination/archive -> /path/to/destination/archive/ - AzFramework::StringFunc::Path::AppendSeparator(destinationPathWithRoot); - return MakePath(destinationPathWithRoot); - } - - AZStd::string GetCreateArchiveCommand(const AZStd::string& archivePath, const AZStd::string& dirToArchive) - { - auto pathCreationResult = MakeCreateArchivePath(archivePath); - if (!pathCreationResult.IsSuccess()) - { - AZ_Error(ErrorChannel, false, pathCreationResult.GetError().c_str()); - return ""; - } - - // LY-116692. Requires proper investigation, the correct format should be: - // AZStd::string::format(CreateArchiveCmd, archivePath.c_str(), dirToArchive.c_str()); - // but unit test ArchiveTest.ListFilesInArchiveBlocking_FilesAtThreeDepths_FilesFound fails - AZ_UNUSED(dirToArchive); - return AZStd::string::format(CreateArchiveCmd, archivePath.c_str()); - } - - AZStd::string GetExtractArchiveCommand(const AZStd::string& archivePath, const AZStd::string& destinationPath, bool includeRoot) - { - auto pathCreationResult = MakeExtractArchivePath(archivePath, destinationPath, includeRoot); - if (!pathCreationResult) - { - AZ_Error(ErrorChannel, false, pathCreationResult.GetError().c_str()); - return ""; - } - - return AZStd::string::format(ExtractArchiveCmd, archivePath.c_str(), pathCreationResult.GetValue().c_str()); - } - - AZStd::string GetAddFilesToArchiveCommand(const AZStd::string& archivePath, const AZStd::string& listFilePath) - { - auto pathCreationResult = MakeCreateArchivePath(archivePath); - if (!pathCreationResult) - { - AZ_Error(ErrorChannel, false, pathCreationResult.GetError().c_str()); - return ""; - } - AZStd::string fileListStr; - - { - AZ::IO::FileIOStream fileStream(listFilePath.c_str(), AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeText); - if (fileStream.IsOpen()) - { - AZ::IO::SizeType length = fileStream.GetLength(); - AZStd::vector charBuffer; - charBuffer.resize_no_construct(length + 1); - - fileStream.Read(length, charBuffer.data()); - charBuffer.back() = 0; - - fileListStr.append("\""); - fileListStr.insert(1, charBuffer.data()); - AzFramework::StringFunc::Replace(fileListStr, "\n", "\" \""); - fileListStr.append("\""); - } - else - { - AZ_Error(ErrorChannel, false, "Unable to read list file ( %s ) \n", listFilePath.c_str()); - return ""; - } - } - - return AZStd::string::format(AddFilesCmd, archivePath.c_str(), fileListStr.c_str()); - } - - AZStd::string GetAddFileToArchiveCommand(const AZStd::string& archivePath, const AZStd::string& file) - { - auto pathCreationResult = MakeCreateArchivePath(archivePath); - if (!pathCreationResult) - { - AZ_Error(ErrorChannel, false, pathCreationResult.GetError().c_str()); - return ""; - } - - return AZStd::string::format(AddFileCmd, archivePath.c_str(), file.c_str()); - } - - AZStd::string GetExtractFileCommand(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite) - { - AZStd::string commandLineArgs; - if (destinationPath.empty()) - { - // Extract file in archive from archive path to the current directory, overwriting a file of the same name that exists there. - commandLineArgs = AZStd::string::format(ExtractFileCmd, overWrite ? ExtractOverwrite : ExtractSkipExisting, archivePath.c_str(), fileInArchive.c_str()); - } - else - { - if (!MakePath(destinationPath).IsSuccess()) - { - AZ_Error(ErrorChannel, false, "Unable to make path ( %s ).\n", destinationPath.c_str()); - return ""; - } - // Extract file in archive from archive path to destinationPath, overwriting a file of the same name that exists there. - commandLineArgs = AZStd::string::format(ExtractFileDestination, overWrite ? ExtractOverwrite : ExtractSkipExisting, archivePath.c_str(), fileInArchive.c_str(), destinationPath.c_str()); - } - - return commandLineArgs; - } - - AZStd::string GetListFilesInArchiveCommand(const AZStd::string& archivePath) - { - AZStd::string commandLineArgs = AZStd::string::format(ListFilesInArchiveCmd, archivePath.c_str()); - return commandLineArgs; - } - - /* - Sample Console Output of the unzip list command - - Archive: /var/folders/1q/12nyzqc913qgm532y2c98mnm6w4_qv/T/ArchiveTests-ra8oMy/TestArchive.pak - Length Date Time Name - --------- ---------- ----- ---- - 0 10-14-2019 15:22 testfolder/ - 1 10-14-2019 15:22 testfolder/folderfile.txt - 1 10-14-2019 15:22 basicfile.txt - 1 10-14-2019 15:22 basicfile2.txt - 0 10-14-2019 15:22 testfolder2/ - 1 10-14-2019 15:22 testfolder2/sharedfolderfile2.txt - 1 10-14-2019 15:22 testfolder2/sharedfolderfile.txt - 0 10-14-2019 15:22 testfolder3/ - 0 10-14-2019 15:22 testfolder3/testfolder4/ - 1 10-14-2019 15:22 testfolder3/testfolder4/depthfile.bat - --------- ------- - 6 10 files - */ - - void ParseConsoleOutputFromListFilesInArchive(const AZStd::string& consoleOutput, AZStd::vector& fileEntries) - { - AZStd::vector fileEntryData; - AzFramework::StringFunc::Tokenize(consoleOutput.c_str(), fileEntryData, "\n"); - int startingLineIdx = 3; // first line that might contain the file name - for (size_t lineIdx = startingLineIdx; lineIdx < fileEntryData.size(); ++lineIdx) - { - AZStd::string& line = fileEntryData[lineIdx]; - AZStd::vector lineEntryData; - AzFramework::StringFunc::Tokenize(line.c_str(), lineEntryData, " "); - AZStd::string& fileName = lineEntryData.back(); - - if(fileName.back() == AZ_CORRECT_FILESYSTEM_SEPARATOR) - { - // if the filename ends with a separator - // than it indicates that this is a directory - continue; - } - - if(fileName.compare("-------") == 0) - { - return; - } - - fileEntries.emplace_back(fileName); - } - } - - } // namespace Platform -} // namespace AzToolsFramework diff --git a/Code/Framework/AzToolsFramework/Platform/Mac/platform_mac_files.cmake b/Code/Framework/AzToolsFramework/Platform/Mac/platform_mac_files.cmake index 9928584f86..c2c5a11c4c 100644 --- a/Code/Framework/AzToolsFramework/Platform/Mac/platform_mac_files.cmake +++ b/Code/Framework/AzToolsFramework/Platform/Mac/platform_mac_files.cmake @@ -7,5 +7,4 @@ # set(FILES - AzToolsFramework/Archive/ArchiveComponent_Mac.cpp ) diff --git a/Code/Framework/AzToolsFramework/Platform/Windows/AzToolsFramework/Archive/ArchiveComponent_Windows.cpp b/Code/Framework/AzToolsFramework/Platform/Windows/AzToolsFramework/Archive/ArchiveComponent_Windows.cpp deleted file mode 100644 index 20f53af18b..0000000000 --- a/Code/Framework/AzToolsFramework/Platform/Windows/AzToolsFramework/Archive/ArchiveComponent_Windows.cpp +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) Contributors to the Open 3D Engine Project. - * For complete copyright and license terms please see the LICENSE at the root of this distribution. - * - * SPDX-License-Identifier: Apache-2.0 OR MIT - * - */ - -#include -#include -#include -#include - -namespace AzToolsFramework -{ - namespace Platform - { - const char CreateArchiveCmd[] = R"(a -tzip -mx=1 "%s" -r "%s\*")"; - - // -aos is for skipping extract on existing files - const char ExtractArchiveCmd[] = R"(x -mmt=off "%s" -o"%s\*" -aos)"; - const char ExtractArchiveWithoutRootCmd[] = R"(x -mmt=off "%s" -o"%s" -aos)"; - const char AddFilesCmd[] = R"(a -tzip "%s" @"%s")"; - const char AddFileCmd[] = R"(a -tzip "%s" "%s")"; - const char ExtractFileCmd[] = R"(e -mmt=off "%s" "%s" %s)"; - const char ExtractFileDestination[] = R"(e -mmt=off "%s" -o"%s" "%s" %s)"; - const char ExtractOverwrite[] = "-aoa"; - const char ExtractSkipExisting[] = "-aos"; - const char ListFilesInArchiveCmd[] = R"(l -r -slt "%s")"; - - AZStd::string Get7zExePath() - { - const char* rootPath = nullptr; - AZ::ComponentApplicationBus::BroadcastResult(rootPath, &AZ::ComponentApplicationRequests::GetEngineRoot); - AZStd::string exePath; - AzFramework::StringFunc::Path::ConstructFull(rootPath, "Tools", "7za", ".exe", exePath); - return exePath; - } - - AZStd::string GetZipExePath() - { - return Get7zExePath(); - } - - AZStd::string GetUnzipExePath() - { - return Get7zExePath(); - } - - AZStd::string GetCreateArchiveCommand(const AZStd::string& archivePath, const AZStd::string& dirToArchive) - { - return AZStd::string::format(CreateArchiveCmd, archivePath.c_str(), dirToArchive.c_str()); - } - - AZStd::string GetExtractArchiveCommand(const AZStd::string& archivePath, const AZStd::string& destinationPath, bool includeRoot) - { - if (includeRoot) - { - // Extract archive path to destinationPath\ and skipping extracting of existing files - return AZStd::string::format(ExtractArchiveCmd, archivePath.c_str(), destinationPath.c_str()); - } - else - { - // Extract archive path to destinationPath and skipping extracting of existing files - return AZStd::string::format(ExtractArchiveWithoutRootCmd, archivePath.c_str(), destinationPath.c_str()); - } - } - - AZStd::string GetAddFilesToArchiveCommand(const AZStd::string& archivePath, const AZStd::string& listFilePath) - { - return AZStd::string::format(AddFilesCmd, archivePath.c_str(), listFilePath.c_str()); - } - - AZStd::string GetAddFileToArchiveCommand(const AZStd::string& archivePath, const AZStd::string& file) - { - return AZStd::string::format(AddFileCmd, archivePath.c_str(), file.c_str()); - } - - AZStd::string GetExtractFileCommand(const AZStd::string& archivePath, const AZStd::string& fileInArchive, const AZStd::string& destinationPath, bool overWrite) - { - AZStd::string commandLineArgs; - if (destinationPath.empty()) - { - // Extract file in archive from archive path to the current directory, overwriting a file of the same name that exists there. - commandLineArgs = AZStd::string::format(ExtractFileCmd, archivePath.c_str(), fileInArchive.c_str(), overWrite ? ExtractOverwrite : ExtractSkipExisting); - } - else - { - // Extract file in archive from archive path to destinationPath, overwriting a file of the same name that exists there. - commandLineArgs = AZStd::string::format(ExtractFileDestination, archivePath.c_str(), destinationPath.c_str(), fileInArchive.c_str(), overWrite ? ExtractOverwrite : ExtractSkipExisting); - } - - return commandLineArgs; - } - - AZStd::string GetListFilesInArchiveCommand(const AZStd::string& archivePath) - { - AZStd::string commandLineArgs = AZStd::string::format(ListFilesInArchiveCmd, archivePath.c_str()); - return commandLineArgs; - } - - /* - File output for our list archive commands takes the following two patterns for files vs directories: - - Path = basicfile2.txt - Folder = - - Size = 1 - Packed Size = 1 - Modified = 2019-03-26 18:31:10 - Created = 2019-03-26 18:31:10 - Accessed = 2019-03-26 18:31:10 - Attributes = A - Encrypted = - - Comment = - CRC = 32D70693 - Method = Store - Characteristics = NTFS - Host OS = FAT - Version = 10 - Volume Index = 0 - Offset = 44 - - Path = testfolder - Folder = + - Size = 0 - Packed Size = 0 - Modified = 2019-03-26 18:31:10 - Created = 2019-03-26 18:31:10 - Accessed = 2019-03-26 18:31:10 - Attributes = D - Encrypted = - - Comment = - CRC = - Method = Store - Characteristics = NTFS - Host OS = FAT - Version = 20 - Volume Index = 0 - Offset = 89 - - */ - - void ParseConsoleOutputFromListFilesInArchive(const AZStd::string& consoleOutput, AZStd::vector& fileEntries) - { - AZStd::vector fileEntryData; - AzFramework::StringFunc::Tokenize(consoleOutput.c_str(), fileEntryData, "\r\n"); - for (size_t slotNum = 0; slotNum < fileEntryData.size(); ++slotNum) - { - AZStd::string& line = fileEntryData[slotNum]; - if (AzFramework::StringFunc::StartsWith(line, "Path = ")) - { - if ((slotNum + 1) < fileEntryData.size()) - { - // We're checking one past each entry we find for the Folder entry and skipping anything marked as a folder - // See sample output above - if (AzFramework::StringFunc::StartsWith(fileEntryData[slotNum + 1], "Folder = -")) - { - AzFramework::StringFunc::Replace(line, "Path = ", "", false, true); - fileEntries.emplace_back(AZStd::move(line)); - slotNum++; - } - } - } - } - } - } // namespace Platform -} // namespace AzToolsFramework diff --git a/Code/Framework/AzToolsFramework/Platform/Windows/platform_windows_files.cmake b/Code/Framework/AzToolsFramework/Platform/Windows/platform_windows_files.cmake index d4fc29984c..c2c5a11c4c 100644 --- a/Code/Framework/AzToolsFramework/Platform/Windows/platform_windows_files.cmake +++ b/Code/Framework/AzToolsFramework/Platform/Windows/platform_windows_files.cmake @@ -7,5 +7,4 @@ # set(FILES - AzToolsFramework/Archive/ArchiveComponent_Windows.cpp ) diff --git a/Code/Framework/AzToolsFramework/Tests/ArchiveTests.cpp b/Code/Framework/AzToolsFramework/Tests/ArchiveTests.cpp index e7dad4b72f..66520c804c 100644 --- a/Code/Framework/AzToolsFramework/Tests/ArchiveTests.cpp +++ b/Code/Framework/AzToolsFramework/Tests/ArchiveTests.cpp @@ -31,7 +31,6 @@ namespace UnitTest { namespace { - bool CreateDummyFile(const QString& fullPathToFile, const QString& tempStr = {}) { QFileInfo fi(fullPathToFile); @@ -50,7 +49,7 @@ namespace UnitTest return true; } - class ArchiveTest : + class ArchiveComponentTest : public ::testing::Test { @@ -73,7 +72,12 @@ namespace UnitTest return "Archive"; } - void CreateArchiveFolder( QString archiveFolderName, QStringList fileList ) + QString GetExtractFolderName() + { + return "Extracted"; + } + + void CreateArchiveFolder(QString archiveFolderName, QStringList fileList) { QDir tempPath = QDir(m_tempDir.GetDirectory()).filePath(archiveFolderName); @@ -84,6 +88,14 @@ namespace UnitTest } } + QString CreateArchiveListTextFile() + { + QString listFilePath = QDir(m_tempDir.GetDirectory()).absoluteFilePath("FileList.txt"); + QString textContent = CreateArchiveFileList().join("\n"); + EXPECT_TRUE(CreateDummyFile(listFilePath, textContent)); + return listFilePath; + } + void CreateArchiveFolder() { CreateArchiveFolder(GetArchiveFolderName(), CreateArchiveFileList()); @@ -99,16 +111,24 @@ namespace UnitTest return QDir(m_tempDir.GetDirectory()).filePath(GetArchiveFolderName()); } + QString GetExtractFolder() + { + return QDir(m_tempDir.GetDirectory()).filePath(GetExtractFolderName()); + } + bool CreateArchive() { - bool createResult{ false }; - AzToolsFramework::ArchiveCommandsBus::BroadcastResult(createResult, &AzToolsFramework::ArchiveCommandsBus::Events::CreateArchiveBlocking, GetArchivePath().toStdString().c_str(), GetArchiveFolder().toStdString().c_str()); - return createResult; + std::future createResult; + AzToolsFramework::ArchiveCommandsBus::BroadcastResult(createResult, + &AzToolsFramework::ArchiveCommandsBus::Events::CreateArchive, + GetArchivePath().toUtf8().constData(), GetArchiveFolder().toUtf8().constData()); + bool result = createResult.get(); + return result; } void SetUp() override { - m_app.reset(aznew ToolsTestApplication("ArchiveTest")); + m_app.reset(aznew ToolsTestApplication("ArchiveComponentTest")); m_app->Start(AzFramework::Application::Descriptor()); // Without this, the user settings component would attempt to save on finalize/shutdown. Since the file is // shared across the whole engine, if multiple tests are run in parallel, the saving could cause a crash @@ -132,76 +152,138 @@ namespace UnitTest }; #if AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS - TEST_F(ArchiveTest, DISABLED_CreateArchiveBlocking_FilesAtThreeDepths_ArchiveCreated) + TEST_F(ArchiveComponentTest, DISABLED_CreateArchive_FilesAtThreeDepths_ArchiveCreated) #else - TEST_F(ArchiveTest, CreateArchiveBlocking_FilesAtThreeDepths_ArchiveCreated) + TEST_F(ArchiveComponentTest, CreateArchive_FilesAtThreeDepths_ArchiveCreated) #endif // AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS { EXPECT_TRUE(m_tempDir.IsValid()); CreateArchiveFolder(); + AZ_TEST_START_TRACE_SUPPRESSION; bool createResult = CreateArchive(); + AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; - EXPECT_EQ(createResult, true); + EXPECT_TRUE(createResult); } #if AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS - TEST_F(ArchiveTest, DISABLED_ListFilesInArchiveBlocking_FilesAtThreeDepths_FilesFound) + TEST_F(ArchiveComponentTest, DISABLED_ListFilesInArchive_FilesAtThreeDepths_FilesFound) #else - TEST_F(ArchiveTest, ListFilesInArchiveBlocking_FilesAtThreeDepths_FilesFound) + TEST_F(ArchiveComponentTest, ListFilesInArchive_FilesAtThreeDepths_FilesFound) #endif // AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS { EXPECT_TRUE(m_tempDir.IsValid()); CreateArchiveFolder(); - + + AZ_TEST_START_TRACE_SUPPRESSION; EXPECT_EQ(CreateArchive(), true); AZStd::vector fileList; bool listResult{ false }; - AzToolsFramework::ArchiveCommandsBus::BroadcastResult(listResult, &AzToolsFramework::ArchiveCommandsBus::Events::ListFilesInArchiveBlocking, GetArchivePath().toStdString().c_str(), fileList); + AzToolsFramework::ArchiveCommandsBus::BroadcastResult(listResult, + &AzToolsFramework::ArchiveCommandsBus::Events::ListFilesInArchive, + GetArchivePath().toUtf8().constData(), fileList); + AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; + EXPECT_TRUE(listResult); EXPECT_EQ(fileList.size(), 6); } #if AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS - TEST_F(ArchiveTest, DISABLED_CreateDeltaCatalog_AssetsNotRegistered_Failure) + TEST_F(ArchiveComponentTest, DISABLED_CreateDeltaCatalog_AssetsNotRegistered_Failure) #else - TEST_F(ArchiveTest, CreateDeltaCatalog_AssetsNotRegistered_Failure) + TEST_F(ArchiveComponentTest, CreateDeltaCatalog_AssetsNotRegistered_Failure) #endif // AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS { QStringList fileList = CreateArchiveFileList(); CreateArchiveFolder(GetArchiveFolderName(), fileList); - + AZ_TEST_START_TRACE_SUPPRESSION; bool createResult = CreateArchive(); + AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; EXPECT_EQ(createResult, true); bool catalogCreated{ true }; AZ::Test::AssertAbsorber assertAbsorber; - AzToolsFramework::AssetBundleCommandsBus::BroadcastResult(catalogCreated, &AzToolsFramework::AssetBundleCommandsBus::Events::CreateDeltaCatalog, GetArchivePath().toStdString().c_str(), true); + AzToolsFramework::AssetBundleCommandsBus::BroadcastResult(catalogCreated, + &AzToolsFramework::AssetBundleCommandsBus::Events::CreateDeltaCatalog, GetArchivePath().toUtf8().constData(), true); EXPECT_EQ(catalogCreated, false); } #if AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS - TEST_F(ArchiveTest, DISABLED_CreateDeltaCatalog_ArchiveWithoutCatalogAssetsRegistered_Success) + TEST_F(ArchiveComponentTest, DISABLED_AddFilesToArchive_FromListFile_Success) +#else + TEST_F(ArchiveComponentTest, AddFilesToArchive_FromListFile_Success) +#endif // AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS + { + QString listFile = CreateArchiveListTextFile(); + CreateArchiveFolder(GetArchiveFolderName(), CreateArchiveFileList()); + + AZ_TEST_START_TRACE_SUPPRESSION; + std::future addResult; + AzToolsFramework::ArchiveCommandsBus::BroadcastResult( + addResult, &AzToolsFramework::ArchiveCommandsBus::Events::AddFilesToArchive, GetArchivePath().toUtf8().constData(), + GetArchiveFolder().toUtf8().constData(), listFile.toUtf8().constData()); + bool result = addResult.get(); + AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; + + EXPECT_TRUE(result); + } + +#if AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS + TEST_F(ArchiveComponentTest, DISABLED_ExtractArchive_AllFiles_Success) +#else + TEST_F(ArchiveComponentTest, ExtractArchive_AllFiles_Success) +#endif // AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS + { + CreateArchiveFolder(); + AZ_TEST_START_TRACE_SUPPRESSION; + bool createResult = CreateArchive(); + AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; + EXPECT_TRUE(createResult); + + AZ_TEST_START_TRACE_SUPPRESSION; + std::future extractResult; + AzToolsFramework::ArchiveCommandsBus::BroadcastResult( + extractResult, &AzToolsFramework::ArchiveCommandsBus::Events::ExtractArchive, GetArchivePath().toUtf8().constData(), + GetExtractFolder().toUtf8().constData()); + bool result = extractResult.get(); + AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; + + EXPECT_TRUE(result); + + QStringList archiveFiles = CreateArchiveFileList(); + for (const auto& file : archiveFiles) + { + QString fullFilePath = QDir(GetExtractFolder()).absoluteFilePath(file); + QFileInfo fi(fullFilePath); + EXPECT_TRUE(fi.exists()); + } + } + +#if AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS + TEST_F(ArchiveComponentTest, DISABLED_CreateDeltaCatalog_ArchiveWithoutCatalogAssetsRegistered_Success) #else - TEST_F(ArchiveTest, CreateDeltaCatalog_ArchiveWithoutCatalogAssetsRegistered_Success) + TEST_F(ArchiveComponentTest, CreateDeltaCatalog_ArchiveWithoutCatalogAssetsRegistered_Success) #endif // AZ_TRAIT_DISABLE_FAILED_ARCHIVE_TESTS { QStringList fileList = CreateArchiveFileList(); CreateArchiveFolder(GetArchiveFolderName(), fileList); + AZ_TEST_START_TRACE_SUPPRESSION; bool createResult = CreateArchive(); + AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; EXPECT_EQ(createResult, true); for (const auto& thisPath : fileList) { AZ::Data::AssetInfo newInfo; - newInfo.m_relativePath = thisPath.toStdString().c_str(); + newInfo.m_relativePath = thisPath.toUtf8().constData(); newInfo.m_assetType = AZ::Uuid::CreateRandom(); newInfo.m_sizeBytes = 100; // Arbitrary AZ::Data::AssetId generatedID(AZ::Uuid::CreateRandom()); @@ -212,7 +294,7 @@ namespace UnitTest bool catalogCreated{ false }; AZ_TEST_START_TRACE_SUPPRESSION; - AzToolsFramework::AssetBundleCommandsBus::BroadcastResult(catalogCreated, &AzToolsFramework::AssetBundleCommandsBus::Events::CreateDeltaCatalog, GetArchivePath().toStdString().c_str(), true); + AzToolsFramework::AssetBundleCommandsBus::BroadcastResult(catalogCreated, &AzToolsFramework::AssetBundleCommandsBus::Events::CreateDeltaCatalog, GetArchivePath().toUtf8().constData(), true); AZ_TEST_STOP_TRACE_SUPPRESSION_NO_COUNT; // produces different counts in different platforms EXPECT_EQ(catalogCreated, true); } diff --git a/Code/Framework/GridMate/GridMate/Carrier/Carrier.cpp b/Code/Framework/GridMate/GridMate/Carrier/Carrier.cpp index 095f392501..2922cc2891 100644 --- a/Code/Framework/GridMate/GridMate/Carrier/Carrier.cpp +++ b/Code/Framework/GridMate/GridMate/Carrier/Carrier.cpp @@ -1316,7 +1316,7 @@ CarrierThread::CarrierThread(const CarrierDesc& desc, AZStd::shared_ptrGetReadablePlatformList(seed).c_str()); m_additionalSeedInfoMap[seed.m_assetId].reset(new AdditionalSeedInfo(assetInfo.m_relativePath.c_str(), platformList)); diff --git a/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.cpp b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.cpp index eb6cc4033c..42a43ef84e 100644 --- a/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.cpp +++ b/Code/Tools/AssetProcessor/AssetBuilder/AssetBuilderComponent.cpp @@ -411,7 +411,7 @@ bool AssetBuilderComponent::RunInResidentMode() m_running = true; m_jobThreadDesc.m_name = "Builder Job Thread"; - m_jobThread = AZStd::thread(AZStd::bind(&AssetBuilderComponent::JobThread, this), &m_jobThreadDesc); + m_jobThread = AZStd::thread(m_jobThreadDesc, AZStd::bind(&AssetBuilderComponent::JobThread, this)); AzFramework::EngineConnectionEvents::Bus::Handler::BusConnect(); // Listen for disconnects diff --git a/Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp b/Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp index 4257457291..5d2d0e77d3 100644 --- a/Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp +++ b/Code/Tools/AssetProcessor/native/tests/AssetCatalog/AssetCatalogUnitTests.cpp @@ -1063,10 +1063,10 @@ namespace AssetProcessor AZStd::thread_desc threadDesc; threadDesc.m_name = "AssetCatalog Thread"; - AZStd::thread catalogThread([this]() + AZStd::thread catalogThread(threadDesc, [this]() { m_data->m_assetCatalog->BuildRegistry(); - }, &threadDesc + } ); AssetNotificationMessage message("some/path/image.png", AssetNotificationMessage::NotificationType::AssetChanged, AZ::Data::AssetType::CreateRandom(), "pc"); diff --git a/Code/Tools/AssetProcessor/native/utilities/AssetServerHandler.cpp b/Code/Tools/AssetProcessor/native/utilities/AssetServerHandler.cpp index 3d87c11488..3fa4b1cacd 100644 --- a/Code/Tools/AssetProcessor/native/utilities/AssetServerHandler.cpp +++ b/Code/Tools/AssetProcessor/native/utilities/AssetServerHandler.cpp @@ -74,8 +74,11 @@ namespace AssetProcessor AZ_TracePrintf(AssetProcessor::DebugChannel, "Extracting archive for job (%s, %s, %s) with fingerprint (%u).\n", builderParams.m_rcJob->GetJobEntry().m_pathRelativeToWatchFolder.toUtf8().data(), builderParams.m_rcJob->GetJobKey().toUtf8().data(), builderParams.m_rcJob->GetPlatformInfo().m_identifier.c_str(), builderParams.m_rcJob->GetOriginalFingerprint()); - bool success = false; - AzToolsFramework::ArchiveCommands::Bus::BroadcastResult(success, &AzToolsFramework::ArchiveCommands::ExtractArchiveBlocking, archiveAbsFilePath.toUtf8().data(), builderParams.GetTempJobDirectory(), false); + std::future extractResult; + AzToolsFramework::ArchiveCommandsBus::BroadcastResult(extractResult, + &AzToolsFramework::ArchiveCommandsBus::Events::ExtractArchive, + archiveAbsFilePath.toUtf8().data(), builderParams.GetTempJobDirectory()); + bool success = extractResult.get(); AZ_Error(AssetProcessor::DebugChannel, success, "Extracting archive operation failed.\n"); return success; } @@ -106,12 +109,15 @@ namespace AssetProcessor return false; } - bool success = false; - AZ_TracePrintf(AssetProcessor::DebugChannel, "Creating archive for job (%s, %s, %s) with fingerprint (%u).\n", builderParams.m_rcJob->GetJobEntry().m_pathRelativeToWatchFolder.toUtf8().data(), builderParams.m_rcJob->GetJobKey().toUtf8().data(), builderParams.m_rcJob->GetPlatformInfo().m_identifier.c_str(), builderParams.m_rcJob->GetOriginalFingerprint()); - AzToolsFramework::ArchiveCommands::Bus::BroadcastResult(success, &AzToolsFramework::ArchiveCommands::CreateArchiveBlocking, archiveAbsFilePath.toUtf8().data(), builderParams.GetTempJobDirectory()); + + std::future createResult; + AzToolsFramework::ArchiveCommandsBus::BroadcastResult(createResult, + &AzToolsFramework::ArchiveCommandsBus::Events::CreateArchive, + archiveAbsFilePath.toUtf8().data(), builderParams.GetTempJobDirectory()); + bool success = createResult.get(); AZ_Error(AssetProcessor::DebugChannel, success, "Creating archive operation failed. \n"); if (success && sourceFileList.size()) @@ -137,14 +143,16 @@ namespace AssetProcessor allSuccess = false; continue; } - bool success{ false }; - AzToolsFramework::ArchiveCommands::Bus::BroadcastResult(success, &AzToolsFramework::ArchiveCommands::AddFileToArchiveBlocking, archivePath.toUtf8().data(), sourceDir.path().toUtf8().data(), thisProduct.c_str()); + std::future addResult; + AzToolsFramework::ArchiveCommandsBus::BroadcastResult(addResult, + &AzToolsFramework::ArchiveCommandsBus::Events::AddFileToArchive, + archivePath.toUtf8().data(), sourceDir.path().toUtf8().data(), thisProduct.c_str()); + bool success = addResult.get(); if (!success) { AZ_Warning(AssetProcessor::DebugChannel, false, "Failed to add %s to %s", thisProduct.c_str(), archivePath.toUtf8().data()); allSuccess = false; } - } return allSuccess; } diff --git a/Code/Tools/RemoteConsole/Core/RemoteConsoleCore.cpp b/Code/Tools/RemoteConsole/Core/RemoteConsoleCore.cpp index 0a8d9d8eb7..f0c1d765e5 100644 --- a/Code/Tools/RemoteConsole/Core/RemoteConsoleCore.cpp +++ b/Code/Tools/RemoteConsole/Core/RemoteConsoleCore.cpp @@ -99,7 +99,7 @@ void SRemoteThreadedObject::Start(const char* name) desc.m_name = name; auto function = AZStd::bind(&SRemoteThreadedObject::ThreadFunction, this); - m_thread = AZStd::thread(function, &desc); + m_thread = AZStd::thread(desc, function); } void SRemoteThreadedObject::WaitForThread() diff --git a/Gems/Atom/RHI/Code/Source/RHI/AsyncWorkQueue.cpp b/Gems/Atom/RHI/Code/Source/RHI/AsyncWorkQueue.cpp index a076bf3e58..61f05306be 100644 --- a/Gems/Atom/RHI/Code/Source/RHI/AsyncWorkQueue.cpp +++ b/Gems/Atom/RHI/Code/Source/RHI/AsyncWorkQueue.cpp @@ -26,7 +26,7 @@ namespace AZ m_workItemIndex = 0; m_lastCompletedWorkItem = AsyncWorkHandle::Null; AZStd::thread_desc threadDesc{ "AsyncWorkQueue" }; - m_thread = AZStd::thread([&]() { ProcessQueue(); }, &threadDesc); + m_thread = AZStd::thread(threadDesc, [&]() { ProcessQueue(); }); m_isInitialized = true; } diff --git a/Gems/Atom/RHI/Code/Source/RHI/CommandQueue.cpp b/Gems/Atom/RHI/Code/Source/RHI/CommandQueue.cpp index b50c36d3f9..f65c36f2ed 100644 --- a/Gems/Atom/RHI/Code/Source/RHI/CommandQueue.cpp +++ b/Gems/Atom/RHI/Code/Source/RHI/CommandQueue.cpp @@ -43,7 +43,7 @@ namespace AZ m_isWorkQueueEmpty = true; AZStd::thread_desc threadDesc{ GetName().GetCStr() }; - m_thread = AZStd::thread([&]() { ProcessQueue(); }, &threadDesc); + m_thread = AZStd::thread(threadDesc, [&]() { ProcessQueue(); }); } return resultCode; } diff --git a/Gems/Atom/RHI/Code/Source/RHI/Fence.cpp b/Gems/Atom/RHI/Code/Source/RHI/Fence.cpp index d030ff8d2b..162b07406e 100644 --- a/Gems/Atom/RHI/Code/Source/RHI/Fence.cpp +++ b/Gems/Atom/RHI/Code/Source/RHI/Fence.cpp @@ -108,7 +108,7 @@ namespace AZ AZStd::thread_desc threadDesc{ "Fence WaitOnCpu Thread" }; - m_waitThread = AZStd::thread([this, callback]() + m_waitThread = AZStd::thread(threadDesc, [this, callback]() { ResultCode resultCode = WaitOnCpu(); if (resultCode != ResultCode::Success) @@ -116,7 +116,7 @@ namespace AZ AZ_Error("Fence", false, "Failed to call WaitOnCpu in async thread."); } callback(); - }, &threadDesc); + }); return ResultCode::Success; } diff --git a/Gems/Atom/RPI/Code/Source/RPI.Public/Shader/ShaderVariantAsyncLoader.cpp b/Gems/Atom/RPI/Code/Source/RPI.Public/Shader/ShaderVariantAsyncLoader.cpp index 4855fbd864..36109b7b82 100644 --- a/Gems/Atom/RPI/Code/Source/RPI.Public/Shader/ShaderVariantAsyncLoader.cpp +++ b/Gems/Atom/RPI/Code/Source/RPI.Public/Shader/ShaderVariantAsyncLoader.cpp @@ -24,12 +24,11 @@ namespace AZ threadDesc.m_name = "ShaderVariantAsyncLoader"; m_serviceThread = AZStd::thread( + threadDesc, [this]() { this->ThreadServiceLoop(); - }, - &threadDesc - ); + }); } void ShaderVariantAsyncLoader::ThreadServiceLoop() diff --git a/Gems/AudioSystem/Code/Source/Engine/AudioSystem.cpp b/Gems/AudioSystem/Code/Source/Engine/AudioSystem.cpp index 5f3603c956..67c5325be9 100644 --- a/Gems/AudioSystem/Code/Source/Engine/AudioSystem.cpp +++ b/Gems/AudioSystem/Code/Source/Engine/AudioSystem.cpp @@ -56,7 +56,7 @@ namespace Audio threadDesc.m_cpuId = AZ_TRAIT_AUDIOSYSTEM_AUDIO_THREAD_AFFINITY; auto threadFunc = AZStd::bind(&CAudioThread::Run, this); - m_thread = AZStd::thread(threadFunc, &threadDesc); + m_thread = AZStd::thread(threadDesc, threadFunc); } /////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/Gems/BarrierInput/Code/Source/BarrierInputClient.cpp b/Gems/BarrierInput/Code/Source/BarrierInputClient.cpp index e6d5399561..ec4cbcfbd1 100644 --- a/Gems/BarrierInput/Code/Source/BarrierInputClient.cpp +++ b/Gems/BarrierInput/Code/Source/BarrierInputClient.cpp @@ -347,7 +347,7 @@ namespace BarrierInput { AZStd::thread_desc threadDesc; threadDesc.m_name = "BarrierInputClientThread"; - m_threadHandle = AZStd::thread(AZStd::bind(&BarrierClient::Run, this), &threadDesc); + m_threadHandle = AZStd::thread(threadDesc, AZStd::bind(&BarrierClient::Run, this)); } //////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/Gems/HttpRequestor/Code/Source/HttpRequestManager.cpp b/Gems/HttpRequestor/Code/Source/HttpRequestManager.cpp index d33f37a2d7..06cf018f76 100644 --- a/Gems/HttpRequestor/Code/Source/HttpRequestManager.cpp +++ b/Gems/HttpRequestor/Code/Source/HttpRequestManager.cpp @@ -37,7 +37,7 @@ namespace HttpRequestor m_runThread = true; AWSNativeSDKInit::InitializationManager::InitAwsApi(); auto function = AZStd::bind(&Manager::ThreadFunction, this); - m_thread = AZStd::thread(function, &desc); + m_thread = AZStd::thread(desc, function); } Manager::~Manager() diff --git a/Gems/LmbrCentral/Code/Source/Builders/LevelBuilder/LevelBuilderWorker.cpp b/Gems/LmbrCentral/Code/Source/Builders/LevelBuilder/LevelBuilderWorker.cpp index 95a908b20c..631bc41358 100644 --- a/Gems/LmbrCentral/Code/Source/Builders/LevelBuilder/LevelBuilderWorker.cpp +++ b/Gems/LmbrCentral/Code/Source/Builders/LevelBuilder/LevelBuilderWorker.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -155,35 +156,17 @@ namespace LevelBuilder { PopulateOptionalLevelDependencies(sourceRelativeFile, productPathDependencies); - AZStd::binary_semaphore extractionCompleteSemaphore; - auto extractResponseLambda = [&]([[maybe_unused]] bool success) { - AZStd::string levelsubfolder; - AzFramework::StringFunc::Path::Join(tempDirectory.c_str(), "level", levelsubfolder); + std::future extractResult; + AzToolsFramework::ArchiveCommandsBus::BroadcastResult( + extractResult, &AzToolsFramework::ArchiveCommandsBus::Events::ExtractArchive, levelPakFile, tempDirectory); - PopulateLevelSliceDependencies(levelsubfolder, productDependencies, productPathDependencies); - PopulateMissionDependencies(levelPakFile, levelsubfolder, productPathDependencies); - PopulateLevelAudioControlDependencies(levelPakFile, productPathDependencies); + extractResult.wait(); - extractionCompleteSemaphore.release(); - }; + auto levelsubfolder = AZ::IO::Path(tempDirectory) / "level"; - AZ::Uuid handle = AZ::Uuid::Create(); - AzToolsFramework::ArchiveCommands::Bus::Broadcast( - &AzToolsFramework::ArchiveCommands::ExtractArchive, - levelPakFile, - tempDirectory, - handle, - extractResponseLambda); - - const int archiveExtractSleepMS = 20; - bool extractionCompleted = false; - while (!extractionCompleted) - { - extractionCompleted = extractionCompleteSemaphore.try_acquire_for(AZStd::chrono::milliseconds(archiveExtractSleepMS)); - // When the archive extraction is completed, the response lambda is queued on the the tick bus. - // This loop will keep executing queued events on the tickbus until the response unlocks the semaphore. - AZ::TickBus::ExecuteQueuedEvents(); - } + PopulateLevelSliceDependencies(levelsubfolder.Native(), productDependencies, productPathDependencies); + PopulateMissionDependencies(levelPakFile, levelsubfolder.Native(), productPathDependencies); + PopulateLevelAudioControlDependencies(levelPakFile, productPathDependencies); } AZStd::string GetLastFolderFromPath(const AZStd::string& path) diff --git a/Gems/Microphone/Code/Source/Platform/Windows/MicrophoneSystemComponent_Windows.cpp b/Gems/Microphone/Code/Source/Platform/Windows/MicrophoneSystemComponent_Windows.cpp index c182ec4f3b..e0b1f1ae36 100644 --- a/Gems/Microphone/Code/Source/Platform/Windows/MicrophoneSystemComponent_Windows.cpp +++ b/Gems/Microphone/Code/Source/Platform/Windows/MicrophoneSystemComponent_Windows.cpp @@ -225,7 +225,7 @@ namespace Audio AZStd::thread_desc threadDesc; threadDesc.m_name = "MicrophoneCapture-WASAPI"; auto captureFunc = AZStd::bind(&MicrophoneSystemComponentWindows::RunAudioCapture, this); - m_captureThread = AZStd::thread(captureFunc, &threadDesc); + m_captureThread = AZStd::thread(threadDesc, captureFunc); return true; } diff --git a/Gems/PhysX/Code/Tests/PhysXMultithreadingTest.cpp b/Gems/PhysX/Code/Tests/PhysXMultithreadingTest.cpp index a0013402b1..205ad4f377 100644 --- a/Gems/PhysX/Code/Tests/PhysXMultithreadingTest.cpp +++ b/Gems/PhysX/Code/Tests/PhysXMultithreadingTest.cpp @@ -119,7 +119,7 @@ namespace PhysX void Start(int waitTimeMilliseconds) { m_waitTimeMilliseconds = waitTimeMilliseconds; - m_thread = AZStd::thread(AZStd::bind(&SceneQueryBase::Tick, this), &m_threadDesc); + m_thread = AZStd::thread(m_threadDesc, AZStd::bind(&SceneQueryBase::Tick, this)); } void Join() diff --git a/Gems/SaveData/Code/Source/SaveDataSystemComponent.cpp b/Gems/SaveData/Code/Source/SaveDataSystemComponent.cpp index dad6e24abc..e92df0da68 100644 --- a/Gems/SaveData/Code/Source/SaveDataSystemComponent.cpp +++ b/Gems/SaveData/Code/Source/SaveDataSystemComponent.cpp @@ -172,14 +172,15 @@ namespace SaveData // This is safe access outside the lock guard because we only remove elements from the list // after the thread completion flag has been set to true (see also JoinAllCompletedThreads). - threadCompletionPair->m_thread = AZStd::make_unique([&threadCompleteFlag = threadCompletionPair->m_threadComplete, - dataBuffer = AZStd::move(saveDataBufferParams.dataBuffer), - dataBufferSize = saveDataBufferParams.dataBufferSize, - dataBufferName = saveDataBufferParams.dataBufferName, - onSavedCallback = saveDataBufferParams.callback, - localUserId = saveDataBufferParams.localUserId, - absoluteFilePath, - useTemporaryFile]() + threadCompletionPair->m_thread = AZStd::make_unique(saveThreadDesc, + [&threadCompleteFlag = threadCompletionPair->m_threadComplete, + dataBuffer = AZStd::move(saveDataBufferParams.dataBuffer), + dataBufferSize = saveDataBufferParams.dataBufferSize, + dataBufferName = saveDataBufferParams.dataBufferName, + onSavedCallback = saveDataBufferParams.callback, + localUserId = saveDataBufferParams.localUserId, + absoluteFilePath, + useTemporaryFile]() { SaveDataNotifications::Result result = SaveDataNotifications::Result::ErrorUnspecified; @@ -234,7 +235,7 @@ namespace SaveData // Set the thread completion flag so it will be joined in JoinAllCompletedThreads. threadCompleteFlag = true; - }, &saveThreadDesc); + }); if (waitForCompletion) { @@ -301,9 +302,10 @@ namespace SaveData // This is safe access outside the lock guard because we only remove elements from the list // after the thread completion flag has been set to true (see also JoinAllCompletedThreads). - threadCompletionPair->m_thread = AZStd::make_unique([&threadCompleteFlag = threadCompletionPair->m_threadComplete, - loadDataBufferParams, - absoluteFilePath]() + threadCompletionPair->m_thread = AZStd::make_unique(loadThreadDesc, + [&threadCompleteFlag = threadCompletionPair->m_threadComplete, + loadDataBufferParams, + absoluteFilePath]() { SaveDataNotifications::DataBuffer dataBuffer = nullptr; AZ::u64 dataBufferSize = 0; @@ -352,7 +354,7 @@ namespace SaveData // Set the thread completion flag so it will be joined in JoinAllCompletedThreads. threadCompleteFlag = true; - }, &loadThreadDesc); + }); if (waitForCompletion) { diff --git a/Tools/7za.exe b/Tools/7za.exe deleted file mode 100644 index 8a7a9ab6fb..0000000000 --- a/Tools/7za.exe +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:77613cca716edf68b9d5bab951463ed7fade5bc0ec465b36190a76299c50f117 -size 733696 diff --git a/Tools/7za_legal_notice.txt b/Tools/7za_legal_notice.txt deleted file mode 100644 index 6bcdfb1e31..0000000000 --- a/Tools/7za_legal_notice.txt +++ /dev/null @@ -1,36 +0,0 @@ -Amazon note: Source for 7-zip is hosted at -https://s3-us-west-2.amazonaws.com/ly-legal/LicenseConformance/7-zip/18.05/7z1805-src.7z - - - - 7-Zip Extra - ~~~~~~~~~~~ - License for use and distribution - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Copyright (C) 1999-2015 Igor Pavlov. - - 7-Zip Extra files are under the GNU LGPL license. - - - Notes: - You can use 7-Zip Extra on any computer, including a computer in a commercial - organization. You don't need to register or pay for 7-Zip. - - - GNU LGPL information - -------------------- - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You can receive a copy of the GNU Lesser General Public License from - http://www.gnu.org/ -