Merge pull request systemd#28129 from poettering/userdb-ratelimit

userdbd: make sure we don't exit under pressure

Author: poettering

src/userdb/userdbd-manager.c

@@ -49,7 +49,18 @@ static int on_sigusr2(sd_event_source *s, const struct signalfd_siginfo *si, voi
 
         assert(s);
 
-        (void) start_workers(m, true); /* Workers told us there's more work, let's add one more worker as long as we are below the high watermark */
+        (void) start_workers(m, /* explicit_request=*/ true); /* Workers told us there's more work, let's add one more worker as long as we are below the high watermark */
+        return 0;
+}
+
+static int on_deferred_start_worker(sd_event_source *s, uint64_t usec, void *userdata) {
+        Manager *m = ASSERT_PTR(userdata);
+
+        assert(s);
+
+        m->deferred_start_worker_event_source = sd_event_source_unref(m->deferred_start_worker_event_source);
+
+        (void) start_workers(m, /* explicit_request=*/ false);
         return 0;
 }
 
@@ -71,8 +82,8 @@ int manager_new(Manager **ret) {
         *m = (Manager) {
                 .listen_fd = -EBADF,
                 .worker_ratelimit = {
-                        .interval = 5 * USEC_PER_SEC,
-                        .burst = 50,
+                        .interval = 2 * USEC_PER_SEC,
+                        .burst = 2500,
                 },
         };
 
@@ -111,6 +122,8 @@ Manager* manager_free(Manager *m) {
         set_free(m->workers_fixed);
         set_free(m->workers_dynamic);
 
+        m->deferred_start_worker_event_source = sd_event_source_unref(m->deferred_start_worker_event_source);
+
         sd_event_unref(m->event);
 
         return mfree(m);
@@ -213,10 +226,31 @@ static int start_workers(Manager *m, bool explicit_request) {
                         break;
 
                 if (!ratelimit_below(&m->worker_ratelimit)) {
-                        /* If we keep starting workers too often, let's fail the whole daemon, something is wrong */
-                        sd_event_exit(m->event, EXIT_FAILURE);
 
-                        return log_error_errno(SYNTHETIC_ERRNO(EUCLEAN), "Worker threads requested too frequently, something is wrong.");
+                        /* If we keep starting workers too often but none sticks, let's fail the whole
+                         * daemon, something is wrong */
+                        if (n == 0) {
+                                sd_event_exit(m->event, EXIT_FAILURE);
+                                return log_error_errno(SYNTHETIC_ERRNO(EUCLEAN), "Worker threads requested too frequently, but worker count is zero, something is wrong.");
+                        }
+
+                        /* Otherwise, let's stop spawning more for a while. */
+                        log_warning("Worker threads requested too frequently, not starting new ones for a while.");
+
+                        if (!m->deferred_start_worker_event_source) {
+                                r = sd_event_add_time(
+                                                m->event,
+                                                &m->deferred_start_worker_event_source,
+                                                CLOCK_MONOTONIC,
+                                                ratelimit_end(&m->worker_ratelimit),
+                                                /* accuracy_usec= */ 0,
+                                                on_deferred_start_worker,
+                                                m);
+                                if (r < 0)
+                                        return log_error_errno(r, "Failed to allocate deferred start worker event source: %m");
+                        }
+
+                        break;
                 }
 
                 r = start_one_worker(m);
@@ -281,5 +315,5 @@ int manager_startup(Manager *m) {
         if (setsockopt(m->listen_fd, SOL_SOCKET, SO_RCVTIMEO, TIMEVAL_STORE(LISTEN_TIMEOUT_USEC), sizeof(struct timeval)) < 0)
                 return log_error_errno(errno, "Failed to se SO_RCVTIMEO: %m");
 
-        return start_workers(m, false);
+        return start_workers(m, /* explicit_request= */ false);
 }

src/userdb/userdbd-manager.h

@@ -21,6 +21,8 @@ struct Manager {
         int listen_fd;
 
         RateLimit worker_ratelimit;
+
+        sd_event_source *deferred_start_worker_event_source;
 };
 
 int manager_new(Manager **ret);