diff --git a/sycl/include/sycl/event.hpp b/sycl/include/sycl/event.hpp index 444306d4686a2..33daacbd420aa 100644 --- a/sycl/include/sycl/event.hpp +++ b/sycl/include/sycl/event.hpp @@ -24,6 +24,12 @@ #include // for hash #include // for vector +#ifdef _WIN32 +#include +#endif +// also defined in event_imp.hpp. probably need to move it elsewhere +//#define CP_LOG_EVENT_LIFECYCLE 1 + namespace sycl { inline namespace _V1 { // Forward declaration @@ -58,6 +64,41 @@ class __SYCL_EXPORT event : public detail::OwnerLessBase { event(cl_event ClEvent, const context &SyclContext); #endif +#ifdef CP_LOG_EVENT_LIFECYCLE + // Copy Constructor // event(const event &rhs) = default; + event(const event &rhs) : impl(rhs.impl) { // Calls std::shared_ptr's copy constructor + std::cout << "EVENT: Copy Constructor (of " << this << ") from " << &rhs << " - new impl: " << impl.get() << " (use_count: " << impl.use_count() << ")" << std::endl; + } + + // Move Constructor // event(event &&rhs) = default; + event(event &&rhs) noexcept : impl(std::move(rhs.impl)) { // Calls std::shared_ptr's move constructor + std::cout << "EVENT: Move Constructor (of " << this << ") from " << &rhs << " - new impl: " << impl.get() << " (use_count: " << impl.use_count() << ")" << std::endl; + } + + // Copy Assignment Operator //event &operator=(const event &rhs) = default; + event &operator=(const event &rhs) { + if (this != &rhs) { + impl = rhs.impl; // Calls std::shared_ptr's copy assignment operator + } + std::cout << "EVENT: Copy Assignment (of " << this << ") from " << &rhs << " - new impl: " << impl.get() << " (use_count: " << impl.use_count() << ")" << std::endl; + return *this; + } + + // Move Assignment Operator // event &operator=(event &&rhs) = default; + event &operator=(event &&rhs) noexcept { + if (this != &rhs) { + impl = std::move(rhs.impl); // Calls std::shared_ptr's move assignment operator + } + std::cout << "EVENT: Move Assignment (of " << this << ") from " << &rhs << " - new impl: " << impl.get() << " (use_count: " << impl.use_count() << ")" << std::endl; + return *this; + } + + // Destructor + ~event() { + std::cout << "EVENT: Destructor (of " << this << ") - impl: " << impl.get() << " (use_count: " << impl.use_count() << ")" << std::endl; + } + +#else event(const event &rhs) = default; event(event &&rhs) = default; @@ -65,6 +106,9 @@ class __SYCL_EXPORT event : public detail::OwnerLessBase { event &operator=(const event &rhs) = default; event &operator=(event &&rhs) = default; + + ~event() = default; // CP +#endif bool operator==(const event &rhs) const; diff --git a/sycl/source/CMakeLists.txt b/sycl/source/CMakeLists.txt index 20bf85f1b61ca..8e6d24dc84704 100644 --- a/sycl/source/CMakeLists.txt +++ b/sycl/source/CMakeLists.txt @@ -3,6 +3,9 @@ #2. Use AddLLVM to modify the build and access config options #cmake_policy(SET CMP0057 NEW) #include(AddLLVM) + +set(CMAKE_BUILD_TYPE Debug) + include(CheckLinkerFlag) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/version.rc.in diff --git a/sycl/source/detail/event_impl.cpp b/sycl/source/detail/event_impl.cpp index 4e5899df6bfb1..19fab613869e4 100644 --- a/sycl/source/detail/event_impl.cpp +++ b/sycl/source/detail/event_impl.cpp @@ -25,6 +25,10 @@ #include #endif + + + + namespace sycl { inline namespace _V1 { namespace detail { @@ -43,17 +47,66 @@ void event_impl::initContextIfNeeded() { } event_impl::~event_impl() { + auto Handle = this->getHandle(); + +#ifdef CP_LOG_EVENT_LIFECYCLE + std::cout << "~event_impl destructor of (" << this + << ") event_impl.cpp:54 UREvent: " << Handle << std::endl; +#endif +#ifdef CP_LOG_EARLY_RELEASE + if (MHasBeenReleased == 0xDEADBEEF) + std::cout << "~event_impl MHasBeenReleased is already set to 0xDEADBEEF this: " + << this << " UREvent: " << Handle + << " USMMemcopyCalled: " << sycl::detail::USMMemcopyCalled + << std::endl; + else if(MHasBeenReleased != 0) + std::cout << "~event_impl MHasBeenReleased corrupted? " << std::hex << MHasBeenReleased << std::endl; +#endif + try { - auto Handle = this->getHandle(); - if (Handle) + + + if (Handle){ +#ifdef CP_LOG_EARLY_RELEASE + // CP abuse + // turn on when calling memcpy, off when releasing an event with a URHandle. + // check + if (!sycl::detail::USMMemcopyCalled) { + // put breakpoint here: + std::cout << "GOTCHA!! ~event_impl this: (" << this + << ") UREvent: " << std::hex << Handle + << " USMMemcopyCalled: " << sycl::detail::USMMemcopyCalled + << std::endl; + } + sycl::detail::USMMemcopyCalled = false; // toggle it off +#endif getAdapter()->call(Handle); + + } + MHasBeenReleased = 0xDEADBEEF; } catch (std::exception &e) { __SYCL_REPORT_EXCEPTION_TO_STREAM("exception in ~event_impl", e); } } + void event_impl::waitInternal(bool *Success) { auto Handle = this->getHandle(); + +#ifdef CP_LOG_EARLY_RELEASE + // CP -- this does not trip + if(MHasBeenReleased == 0xDEADBEEF){ + std::cout << "waitInternal HasBeenReleased is already set to " + "0xDEADBEEF. this: " + << this << " UREvent: " << std::hex << Handle + << " USMMemcopyCalled: " << sycl::detail::USMMemcopyCalled + << std::endl; + }else if(MHasBeenReleased != 0){ + std::cout << "waitInternal MHasBeenReleased corrupted? " << std::hex << MHasBeenReleased << std::endl; + } +#endif + + if (!MIsHostEvent && Handle) { // Wait for the native event ur_result_t Err = @@ -145,7 +198,9 @@ void event_impl::setContextImpl(const ContextImplPtr &Context) { event_impl::event_impl(ur_event_handle_t Event, const context &SyclContext) : MEvent(Event), MContext(detail::getSyclObjImpl(SyclContext)), MIsFlushed(true), MState(HES_Complete) { - +#ifdef CP_LOG_EVENT_LIFECYCLE + std::cout << "event_impl constructor. of (" << this << ") event_impl.cpp:178" << std::endl; +#endif ur_context_handle_t TempContext; getAdapter()->call( this->getHandle(), UR_EVENT_INFO_CONTEXT, sizeof(ur_context_handle_t), @@ -161,6 +216,9 @@ event_impl::event_impl(ur_event_handle_t Event, const context &SyclContext) event_impl::event_impl(const QueueImplPtr &Queue) : MQueue{Queue}, MIsProfilingEnabled{!Queue || Queue->MIsProfilingEnabled} { +#ifdef CP_LOG_EVENT_LIFECYCLE + std::cout << "event_impl constructor from QueImplPtr of (" << this << ") event_impl.cpp:197" << std::endl; +#endif if (Queue) this->setContextImpl(Queue->getContextImplPtr()); else { @@ -246,6 +304,16 @@ void event_impl::instrumentationEpilog(void *TelemetryEvent, void event_impl::wait(std::shared_ptr Self, bool *Success) { +#ifdef CP_LOG_EARLY_RELEASE + // CP -- this trips + if(MHasBeenReleased == 0xDEADBEEF) { + std::cout << "wait HasBeenRelease already set to 0xDEADBEEF. this: " << this << std::endl; + //__debugbreak(); + }else if(MHasBeenReleased != 0){ + std::cout << "wait MHasBeenReleased corrupted? " << std::hex << MHasBeenReleased << std::endl; + } +#endif + if (MState == HES_Discarded) throw sycl::exception(make_error_code(errc::invalid), "wait method cannot be used for a discarded event."); diff --git a/sycl/source/detail/event_impl.hpp b/sycl/source/detail/event_impl.hpp index 0297ebfd3e4d3..00ead9bf6bcdb 100644 --- a/sycl/source/detail/event_impl.hpp +++ b/sycl/source/detail/event_impl.hpp @@ -20,6 +20,20 @@ #include #include + +// CP -- enables logging of "early release" and also global var to track memcpy :: ~event_impl tracking +#define CP_LOG_EARLY_RELEASE 1 + +#ifdef _WIN32 +#include +#else +// CP no op +void __debugbreak() { } +#endif +// also in event.hpp. Probably needs to be moved elsewhere +//#define CP_LOG_EVENT_LIFECYCLE 1 + + namespace sycl { inline namespace _V1 { namespace ext::oneapi::experimental::detail { @@ -35,6 +49,11 @@ using QueueImplPtr = std::shared_ptr; class event_impl; using EventImplPtr = std::shared_ptr; +#ifdef CP_LOG_EARLY_RELEASE +// CP Adding a global +bool USMMemcopyCalled = false; +#endif + class event_impl { public: enum HostEventState : int { @@ -56,6 +75,9 @@ class event_impl { // event methods. This ::get() call uses static vars to read and parse the // ODS env var exactly once. SYCLConfig::get(); +#ifdef CP_LOG_EVENT_LIFECYCLE + std::cout << "event_impl ready constructor of (" << this << ") event_impl.hpp:53" << std::endl; +#endif } /// Constructs an event instance from a UR event handle. @@ -67,6 +89,12 @@ class event_impl { /// \param SyclContext is an instance of SYCL context. event_impl(ur_event_handle_t Event, const context &SyclContext); event_impl(const QueueImplPtr &Queue); + + // delete the copy constructors and assignment operators + event_impl(const event_impl&) = delete; + event_impl &operator=(const event_impl&) = delete; + event_impl(event_impl&&) = delete; + event_impl &operator=(event_impl&&) = delete; /// Sets a queue associated with the event /// @@ -347,7 +375,7 @@ class event_impl { // queue and command, as well as the fact that it is not in enqueued state. return MEvent && MQueue.expired() && !MIsEnqueued && !MCommand; } - +uint32_t MHasBeenReleased = 0; protected: // When instrumentation is enabled emits trace event for event wait begin and // returns the telemetry event generated for the wait @@ -433,6 +461,7 @@ class event_impl { // MEvent is lazily created in first ur handle query. bool MIsDefaultConstructed = false; bool MIsHostEvent = false; + }; } // namespace detail diff --git a/sycl/source/detail/memory_manager.cpp b/sycl/source/detail/memory_manager.cpp index a650fb59fcc26..c2bdce480ea48 100644 --- a/sycl/source/detail/memory_manager.cpp +++ b/sycl/source/detail/memory_manager.cpp @@ -902,6 +902,11 @@ void MemoryManager::copy_usm(const void *SrcMem, QueueImplPtr SrcQueue, "NULL pointer argument in memory copy operation."); const AdapterPtr &Adapter = SrcQueue->getAdapter(); + // CP +#ifdef CP_LOG_EARLY_RELEASE + sycl::detail::USMMemcopyCalled = true; // turn on when calling memcpy, off when releasing. Check if imbalance. +#endif + Adapter->call(SrcQueue->getHandleRef(), /* blocking */ false, DstMem, SrcMem, Len, DepEvents.size(), diff --git a/sycl/source/event.cpp b/sycl/source/event.cpp index 5af373c29c17e..e0601b924922a 100644 --- a/sycl/source/event.cpp +++ b/sycl/source/event.cpp @@ -19,14 +19,25 @@ #include #include +#ifdef _WIN32 +#include +#endif + namespace sycl { inline namespace _V1 { -event::event() : impl(std::make_shared(std::nullopt)) {} +event::event() : impl(std::make_shared(std::nullopt)) { +#ifdef CP_LOG_EVENT_LIFECYCLE + std::cout << "EVENT() constructor (of " << this << ") impl: " << impl.get() << " (use_count: " << impl.use_count() << ") event.cpp:25" << std::endl; +#endif +} event::event(cl_event ClEvent, const context &SyclContext) : impl(std::make_shared( detail::ur::cast(ClEvent), SyclContext)) { +#ifdef CP_LOG_EVENT_LIFECYCLE + std::cout << "EVENT(ClEvent, Context) constructor. event.cpp:31" << std::endl; +#endif // This is a special interop constructor for OpenCL, so the event must be // retained. __SYCL_OCL_CALL(clRetainEvent, ClEvent); @@ -62,7 +73,11 @@ std::vector event::get_wait_list() { } event::event(std::shared_ptr event_impl) - : impl(std::move(event_impl)) {} + : impl(std::move(event_impl)) { +#ifdef CP_LOG_EVENT_LIFECYCLE + std::cout << "EVENT(impl) constructor of (" << this << ") impl: " << impl.get() << " (use_count: " << impl.use_count() << ") event.cpp:71" << std::endl; +#endif +} template typename detail::is_event_info_desc::return_type diff --git a/unified-runtime/source/CMakeLists.txt b/unified-runtime/source/CMakeLists.txt index f0dd315313e73..228b35eeef98f 100644 --- a/unified-runtime/source/CMakeLists.txt +++ b/unified-runtime/source/CMakeLists.txt @@ -3,6 +3,8 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +set(CMAKE_BUILD_TYPE Debug) + add_definitions(-DUR_VERSION="${PROJECT_VERSION_MAJOR}") add_definitions(-DUR_VALIDATION_LAYER_SUPPORTED_VERSION="${PROJECT_VERSION_MAJOR}") diff --git a/unified-runtime/source/adapters/level_zero/common.cpp b/unified-runtime/source/adapters/level_zero/common.cpp index bac059c46f170..c3e4af2064b6a 100644 --- a/unified-runtime/source/adapters/level_zero/common.cpp +++ b/unified-runtime/source/adapters/level_zero/common.cpp @@ -353,6 +353,7 @@ getZeStructureType() { thread_local int32_t ErrorMessageCode = 0; thread_local char ErrorMessage[MaxMessageSize]{}; thread_local int32_t ErrorAdapterNativeCode; +bool ReleaseTracker = true; // Utility function for setting a message and warning [[maybe_unused]] void setErrorMessage(const char *pMessage, int32_t ErrorCode, diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index 33a1072e217a9..daaffe6aee540 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -381,3 +381,5 @@ extern thread_local int32_t ErrorAdapterNativeCode; // Utility function for setting a message and warning [[maybe_unused]] void setErrorMessage(const char *pMessage, int32_t ErrorCode, int32_t AdapterErrorCode); + +extern bool ReleaseTracker; \ No newline at end of file diff --git a/unified-runtime/source/adapters/level_zero/event.cpp b/unified-runtime/source/adapters/level_zero/event.cpp index f06cae5ec0cb3..3f94dbb4c0a40 100644 --- a/unified-runtime/source/adapters/level_zero/event.cpp +++ b/unified-runtime/source/adapters/level_zero/event.cpp @@ -793,7 +793,7 @@ urEventWait(uint32_t NumEvents, // ur_event_handle_t_ *Event = ur_cast(e); if (!Event->hasExternalRefs()) - die("urEventWait must not be called for an internal event"); + die("FIRST urEventWait must not be called for an internal event"); ze_event_handle_t ZeHostVisibleEvent; if (auto Res = Event->getOrCreateHostVisibleEvent(ZeHostVisibleEvent)) @@ -819,7 +819,7 @@ urEventWait(uint32_t NumEvents, { std::shared_lock EventLock(Event->Mutex); if (!Event->hasExternalRefs()) - die("urEventWait must not be called for an internal event"); + die("SECOND urEventWait must not be called for an internal event"); if (!Event->Completed) { auto HostVisibleEvent = Event->HostVisibleEvent; @@ -882,6 +882,12 @@ urEventRetain(/** [in] handle of the event object */ ur_event_handle_t Event) { ur_result_t urEventRelease(/** [in] handle of the event object */ ur_event_handle_t Event) { + // The releaseTracker is not a per-class var, just a global + //if (ReleaseTracker) { + //std::cout << "Double release call?" << std::endl; // changes timing. + //__debugbreak(); + //} + ReleaseTracker = true; Event->RefCountExternal--; bool isEventsWaitCompleted = (Event->CommandType == UR_COMMAND_EVENTS_WAIT ||