[L0 v2] implement in-order list without driver support

Author: igchor

source/adapters/level_zero/v2/context.cpp

@@ -53,8 +53,6 @@ ur_context_handle_t_::ur_context_handle_t_(ze_context_handle_t hContext,
                      [context = this, platform = phDevices[0]->Platform](
                          DeviceId deviceId, v2::event_flags_t flags)
                          -> std::unique_ptr<v2::event_provider> {
-                       assert((flags & v2::EVENT_FLAGS_COUNTER) != 0);
-
                        std::ignore = deviceId;
                        std::ignore = platform;
 

source/adapters/level_zero/v2/event.cpp

@@ -95,6 +95,7 @@ ur_event_handle_t_::ur_event_handle_t_(ur_context_handle_t hContext,
 
 void ur_event_handle_t_::resetQueueAndCommand(ur_queue_handle_t hQueue,
                                               ur_command_t commandType) {
+  resetted = false;
   this->hQueue = hQueue;
   this->commandType = commandType;
 }
@@ -124,6 +125,7 @@ void ur_event_handle_t_::reset() {
   if (!(flags & v2::EVENT_FLAGS_COUNTER)) {
     zeEventHostReset(hZeEvent);
   }
+  resetted = true;
 }
 
 ze_event_handle_t ur_event_handle_t_::getZeEvent() const {
@@ -187,7 +189,7 @@ ur_pooled_event_t::ur_pooled_event_t(
       zeEvent(std::move(eventAllocation)), pool(pool) {}
 
 ur_result_t ur_pooled_event_t::forceRelease() {
-  pool->free(this);
+  pool->free(this, false);
   return UR_RESULT_SUCCESS;
 }
 
@@ -376,8 +378,8 @@ urEventCreateWithNativeHandle(ur_native_handle_t hNativeEvent,
                               const ur_event_native_properties_t *pProperties,
                               ur_event_handle_t *phEvent) try {
   if (!hNativeEvent) {
-    assert(hContext->nativeEventsPool.getFlags() &
-           v2::EVENT_FLAGS_COUNTER == 0);
+    assert((hContext->nativeEventsPool.getFlags() & v2::EVENT_FLAGS_COUNTER) ==
+           0);
 
     *phEvent = hContext->nativeEventsPool.allocate();
     ZE2UR_CALL(zeEventHostSignal, ((*phEvent)->getZeEvent()));

source/adapters/level_zero/v2/event.hpp

@@ -97,6 +97,8 @@ struct ur_event_handle_t_ : _ur_object {
   uint64_t getEventStartTimestmap() const;
   uint64_t getEventEndTimestamp();
 
+  bool isResetted() const { return resetted; }
+
 protected:
   ur_context_handle_t hContext;
 
@@ -110,6 +112,8 @@ struct ur_event_handle_t_ : _ur_object {
 
   v2::event_flags_t flags;
   event_profiling_data_t profilingData;
+
+  bool resetted = false;
 };
 
 struct ur_pooled_event_t : ur_event_handle_t_ {

source/adapters/level_zero/v2/event_pool.cpp

@@ -17,6 +17,14 @@ namespace v2 {
 
 static constexpr size_t EVENTS_BURST = 64;
 
+static uint64_t CHECK_EXECUTING = []() {
+  return getenv_to_unsigned("UR_L0_V2_CHECK_EXECUTING").value_or(16);
+}();
+
+static uint64_t EVENT_POOL_CLEANUP_SIZE = []() {
+  return getenv_to_unsigned("UR_L0_V2_EVENT_POOL_CLEANUP_SIZE").value_or(1000);
+}();
+
 ur_pooled_event_t *event_pool::allocate() {
   TRACK_SCOPE_LATENCY("event_pool::allocate");
 
@@ -39,13 +47,53 @@ ur_pooled_event_t *event_pool::allocate() {
   return event;
 }
 
-void event_pool::free(ur_pooled_event_t *event) {
+void event_pool::forceCleanupExecuting() {
+  TRACK_SCOPE_LATENCY("event_pool::forceCleanupExecuting");
+
+  std::unique_lock<std::mutex> lock(*mutex);
+
+  for (auto &event : executing) {
+    event->reset();
+    freelist.push_back(event);
+  }
+
+  executing.clear();
+}
+
+void event_pool::cleanupExecuting() {
+  TRACK_SCOPE_LATENCY("event_pool::cleanupExecuting");
+
+  std::unique_lock<std::mutex> lock(*mutex);
+
+  if (executing.size() < EVENT_POOL_CLEANUP_SIZE) {
+    return;
+  }
+
+  size_t completed = 0;
+  for (size_t i = 0; i < std::min(CHECK_EXECUTING, executing.size()); i++) {
+    if (zeEventQueryStatus(executing[i]->getZeEvent()) == ZE_RESULT_SUCCESS) {
+      executing[i]->reset();
+      freelist.push_back(executing[i]);
+      completed++;
+    } else {
+      break;
+    }
+  }
+
+  executing.erase(executing.begin(), executing.begin() + completed);
+}
+
+void event_pool::free(ur_pooled_event_t *event, bool completed) {
   TRACK_SCOPE_LATENCY("event_pool::free");
 
   std::unique_lock<std::mutex> lock(*mutex);
 
-  event->reset();
-  freelist.push_back(event);
+  if (completed) {
+    event->reset();
+    freelist.push_back(event);
+  } else {
+    executing.push_back(event);
+  }
 
   // The event is still in the pool, so we need to increment the refcount
   assert(event->RefCount.load() == 0);

source/adapters/level_zero/v2/event_pool.hpp

@@ -40,11 +40,22 @@ class event_pool {
   event_pool(const event_pool &) = delete;
   event_pool &operator=(const event_pool &) = delete;
 
+  ~event_pool() {
+    for (auto &event : executing) {
+      zeEventHostSynchronize(event->getZeEvent(), UINT64_MAX);
+    }
+  }
+
   // Allocate an event from the pool. Thread safe.
   ur_pooled_event_t *allocate();
 
   // Free an event back to the pool. Thread safe.
-  void free(ur_pooled_event_t *event);
+  void free(ur_pooled_event_t *event, bool completed);
+
+  // Move events from the executing list to the freelist (if they are
+  // completed). Thread safe.
+  void cleanupExecuting();
+  void forceCleanupExecuting();
 
   event_provider *getProvider() const;
   event_flags_t getFlags() const;
@@ -54,6 +65,7 @@ class event_pool {
   std::unique_ptr<event_provider> provider;
 
   std::deque<ur_pooled_event_t> events;
+  std::deque<ur_pooled_event_t *> executing;
   std::vector<ur_pooled_event_t *> freelist;
 
   std::unique_ptr<std::mutex> mutex;