wayland: Add xdg_toplevel v7 edge constraint support

Kontrabant · Kontrabant · commit edd08771a91d · 2025-04-13T11:10:49.000-04:00
If a window isn't resizable from specific directions, the compositor can inform clients of the current edge constraints, so they don't display resize cursors for non-resizable edges.
diff --git a/src/video/wayland/SDL_waylandevents.c b/src/video/wayland/SDL_waylandevents.c
@@ -599,6 +599,73 @@ static void pointer_handle_motion(void *data, struct wl_pointer *pointer,
 
         if (window->hit_test) {
             SDL_HitTestResult rc = window->hit_test(window, &seat->pointer.last_motion, window->hit_test_data);
+
+            // Apply the toplevel constraints if the window isn't resizable from those directions.
+            switch (rc) {
+            case SDL_HITTEST_RESIZE_TOPLEFT:
+                if ((window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_TOP) &&
+                    (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_LEFT)) {
+                    rc = SDL_HITTEST_NORMAL;
+                } else if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_TOP) {
+                    rc = SDL_HITTEST_RESIZE_LEFT;
+                } else if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_LEFT) {
+                    rc = SDL_HITTEST_RESIZE_TOP;
+                }
+                break;
+            case SDL_HITTEST_RESIZE_TOP:
+                if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_TOP) {
+                    rc = SDL_HITTEST_NORMAL;
+                }
+                break;
+            case SDL_HITTEST_RESIZE_TOPRIGHT:
+                if ((window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_TOP) &&
+                    (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_RIGHT)) {
+                    rc = SDL_HITTEST_NORMAL;
+                } else if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_TOP) {
+                    rc = SDL_HITTEST_RESIZE_RIGHT;
+                } else if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_RIGHT) {
+                    rc = SDL_HITTEST_RESIZE_TOP;
+                }
+                break;
+            case SDL_HITTEST_RESIZE_RIGHT:
+                if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_RIGHT) {
+                    rc = SDL_HITTEST_NORMAL;
+                }
+                break;
+            case SDL_HITTEST_RESIZE_BOTTOMRIGHT:
+                if ((window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_BOTTOM) &&
+                    (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_RIGHT)) {
+                    rc = SDL_HITTEST_NORMAL;
+                } else if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_BOTTOM) {
+                    rc = SDL_HITTEST_RESIZE_RIGHT;
+                } else if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_RIGHT) {
+                    rc = SDL_HITTEST_RESIZE_BOTTOM;
+                }
+                break;
+            case SDL_HITTEST_RESIZE_BOTTOM:
+                if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_BOTTOM) {
+                    rc = SDL_HITTEST_NORMAL;
+                }
+                break;
+            case SDL_HITTEST_RESIZE_BOTTOMLEFT:
+                if ((window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_BOTTOM) &&
+                    (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_LEFT)) {
+                    rc = SDL_HITTEST_NORMAL;
+                } else if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_BOTTOM) {
+                    rc = SDL_HITTEST_RESIZE_LEFT;
+                } else if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_LEFT) {
+                    rc = SDL_HITTEST_RESIZE_BOTTOM;
+                }
+                break;
+            case SDL_HITTEST_RESIZE_LEFT:
+                if (window_data->toplevel_constraints & WAYLAND_TOPLEVEL_CONSTRAINED_LEFT) {
+                    rc = SDL_HITTEST_NORMAL;
+                }
+                break;
+            default:
+                break;
+            }
+
             if (rc != window_data->hit_test_result) {
                 window_data->hit_test_result = rc;
                 Wayland_SeatUpdateCursor(seat);
diff --git a/src/video/wayland/SDL_waylandvideo.c b/src/video/wayland/SDL_waylandvideo.c
@@ -1253,7 +1253,7 @@ static void display_handle_global(void *data, struct wl_registry *registry, uint
         struct wl_seat *seat = wl_registry_bind(d->registry, id, &wl_seat_interface, SDL_min(SDL_WL_SEAT_VERSION, version));
         Wayland_DisplayCreateSeat(d, seat, id);
     } else if (SDL_strcmp(interface, "xdg_wm_base") == 0) {
-        d->shell.xdg = wl_registry_bind(d->registry, id, &xdg_wm_base_interface, SDL_min(version, 6));
+        d->shell.xdg = wl_registry_bind(d->registry, id, &xdg_wm_base_interface, SDL_min(version, 7));
         xdg_wm_base_add_listener(d->shell.xdg, &shell_listener_xdg, NULL);
     } else if (SDL_strcmp(interface, "wl_shm") == 0) {
         d->shm = wl_registry_bind(registry, id, &wl_shm_interface, 1);
diff --git a/src/video/wayland/SDL_waylandwindow.c b/src/video/wayland/SDL_waylandwindow.c
@@ -774,6 +774,7 @@ static void handle_configure_xdg_toplevel(void *data,
     bool active = false;
     bool resizing = false;
     bool suspended = false;
+    wind->toplevel_constraints = 0;
     wl_array_for_each (state, states) {
         switch (*state) {
         case XDG_TOPLEVEL_STATE_FULLSCREEN:
@@ -800,6 +801,18 @@ static void handle_configure_xdg_toplevel(void *data,
         case XDG_TOPLEVEL_STATE_SUSPENDED:
             suspended = true;
             break;
+        case XDG_TOPLEVEL_STATE_CONSTRAINED_LEFT:
+            wind->toplevel_constraints |= WAYLAND_TOPLEVEL_CONSTRAINED_LEFT;
+            break;
+        case XDG_TOPLEVEL_STATE_CONSTRAINED_RIGHT:
+            wind->toplevel_constraints |= WAYLAND_TOPLEVEL_CONSTRAINED_RIGHT;
+            break;
+        case XDG_TOPLEVEL_STATE_CONSTRAINED_TOP:
+            wind->toplevel_constraints |= WAYLAND_TOPLEVEL_CONSTRAINED_TOP;
+            break;
+        case XDG_TOPLEVEL_STATE_CONSTRAINED_BOTTOM:
+            wind->toplevel_constraints |= WAYLAND_TOPLEVEL_CONSTRAINED_BOTTOM;
+            break;
         default:
             break;
         }
@@ -1205,6 +1218,7 @@ static void decoration_frame_configure(struct libdecor_frame *frame,
 #if SDL_LIBDECOR_CHECK_VERSION(0, 3, 0)
         resizing = (window_state & LIBDECOR_WINDOW_STATE_RESIZING) != 0;
 #endif
+        // TODO: Toplevel constraint passthrough is waiting on upstream libdecor changes.
     }
     const bool floating = !(fullscreen || maximized || tiled);
 
diff --git a/src/video/wayland/SDL_waylandwindow.h b/src/video/wayland/SDL_waylandwindow.h
@@ -95,6 +95,13 @@ struct SDL_WindowData
                               WAYLAND_WM_CAPS_FULLSCREEN |
                               WAYLAND_WM_CAPS_MINIMIZE
     } wm_caps;
+    enum
+    {
+        WAYLAND_TOPLEVEL_CONSTRAINED_LEFT = 0x01,
+        WAYLAND_TOPLEVEL_CONSTRAINED_RIGHT = 0x02,
+        WAYLAND_TOPLEVEL_CONSTRAINED_TOP = 0x04,
+        WAYLAND_TOPLEVEL_CONSTRAINED_BOTTOM = 0x08
+    } toplevel_constraints;
 
     struct wl_egl_window *egl_window;
 #ifdef SDL_VIDEO_OPENGL_EGL
diff --git a/wayland-protocols/xdg-shell.xml b/wayland-protocols/xdg-shell.xml
@@ -29,7 +29,7 @@
     DEALINGS IN THE SOFTWARE.
   </copyright>
 
-  <interface name="xdg_wm_base" version="6">
+  <interface name="xdg_wm_base" version="7">
     <description summary="create desktop-style surfaces">
       The xdg_wm_base interface is exposed as a global object enabling clients
       to turn their wl_surfaces into windows in a desktop environment. It
@@ -122,7 +122,7 @@
     </event>
   </interface>
 
-  <interface name="xdg_positioner" version="6">
+  <interface name="xdg_positioner" version="7">
     <description summary="child surface positioner">
       The xdg_positioner provides a collection of rules for the placement of a
       child surface relative to a parent surface. Rules can be defined to ensure
@@ -344,7 +344,7 @@
 
 	The default adjustment is none.
       </description>
-      <arg name="constraint_adjustment" type="uint"
+      <arg name="constraint_adjustment" type="uint" enum="constraint_adjustment"
 	   summary="bit mask of constraint adjustments"/>
     </request>
 
@@ -407,7 +407,7 @@
     </request>
   </interface>
 
-  <interface name="xdg_surface" version="6">
+  <interface name="xdg_surface" version="7">
     <description summary="desktop user interface surface base interface">
       An interface that may be implemented by a wl_surface, for
       implementations that provide a desktop-style user interface.
@@ -434,7 +434,8 @@
       manipulate a buffer prior to the first xdg_surface.configure call must
       also be treated as errors.
 
-      After creating a role-specific object and setting it up, the client must
+      After creating a role-specific object and setting it up (e.g. by sending
+      the title, app ID, size constraints, parent, etc), the client must
       perform an initial commit without any buffer attached. The compositor
       will reply with initial wl_surface state such as
       wl_surface.preferred_buffer_scale followed by an xdg_surface.configure
@@ -515,8 +516,7 @@
 	portions like drop-shadows which should be ignored for the
 	purposes of aligning, placing and constraining windows.
 
-	The window geometry is double buffered, and will be applied at the
-	time wl_surface.commit of the corresponding wl_surface is called.
+	The window geometry is double-buffered state, see wl_surface.commit.
 
 	When maintaining a position, the compositor should treat the (x, y)
 	coordinate of the window geometry as the top left corner of the window.
@@ -617,21 +617,25 @@
 
   </interface>
 
-  <interface name="xdg_toplevel" version="6">
+  <interface name="xdg_toplevel" version="7">
     <description summary="toplevel surface">
       This interface defines an xdg_surface role which allows a surface to,
       among other things, set window-like properties such as maximize,
       fullscreen, and minimize, set application-specific metadata like title and
       id, and well as trigger user interactive operations such as interactive
       resize and move.
 
+      A xdg_toplevel by default is responsible for providing the full intended
+      visual representation of the toplevel, which depending on the window
+      state, may mean things like a title bar, window controls and drop shadow.
+
       Unmapping an xdg_toplevel means that the surface cannot be shown
       by the compositor until it is explicitly mapped again.
       All active operations (e.g., move, resize) are canceled and all
       attributes (e.g. title, state, stacking, ...) are discarded for
       an xdg_toplevel surface when it is unmapped. The xdg_toplevel returns to
       the state it had right after xdg_surface.get_toplevel. The client
-      can re-map the toplevel by perfoming a commit without any buffer
+      can re-map the toplevel by performing a commit without any buffer
       attached, waiting for a configure event and handling it as usual (see
       xdg_surface description).
 
@@ -828,8 +832,7 @@
 	configure event to ensure that both the client and the compositor
 	setting the state can be synchronized.
 
-	States set in this way are double-buffered. They will get applied on
-	the next commit.
+	States set in this way are double-buffered, see wl_surface.commit.
       </description>
       <entry name="maximized" value="1" summary="the surface is maximized">
 	<description summary="the surface is maximized">
@@ -869,24 +872,36 @@
 	<description summary="the surface’s left edge is tiled">
 	  The window is currently in a tiled layout and the left edge is
 	  considered to be adjacent to another part of the tiling grid.
+
+	  The client should draw without shadow or other decoration outside of
+	  the window geometry on the left edge.
 	</description>
       </entry>
       <entry name="tiled_right" value="6" since="2">
 	<description summary="the surface’s right edge is tiled">
 	  The window is currently in a tiled layout and the right edge is
 	  considered to be adjacent to another part of the tiling grid.
+
+	  The client should draw without shadow or other decoration outside of
+	  the window geometry on the right edge.
 	</description>
       </entry>
       <entry name="tiled_top" value="7" since="2">
 	<description summary="the surface’s top edge is tiled">
 	  The window is currently in a tiled layout and the top edge is
 	  considered to be adjacent to another part of the tiling grid.
+
+	  The client should draw without shadow or other decoration outside of
+	  the window geometry on the top edge.
 	</description>
       </entry>
       <entry name="tiled_bottom" value="8" since="2">
 	<description summary="the surface’s bottom edge is tiled">
 	  The window is currently in a tiled layout and the bottom edge is
 	  considered to be adjacent to another part of the tiling grid.
+
+	  The client should draw without shadow or other decoration outside of
+	  the window geometry on the bottom edge.
 	</description>
       </entry>
       <entry name="suspended" value="9" since="6">
@@ -896,6 +911,38 @@
 	  outputs are switched off due to screen locking.
 	</description>
       </entry>
+      <entry name="constrained_left" value="10" since="7">
+	<description summary="the surface’s left edge is constrained">
+          The left edge of the window is currently constrained, meaning it
+          shouldn't attempt to resize from that edge. It can for example mean
+          it's tiled next to a monitor edge on the constrained side of the
+          window.
+	</description>
+      </entry>
+      <entry name="constrained_right" value="11" since="7">
+	<description summary="the surface’s right edge is constrained">
+          The right edge of the window is currently constrained, meaning it
+          shouldn't attempt to resize from that edge. It can for example mean
+          it's tiled next to a monitor edge on the constrained side of the
+          window.
+	</description>
+      </entry>
+      <entry name="constrained_top" value="12" since="7">
+	<description summary="the surface’s top edge is constrained">
+          The top edge of the window is currently constrained, meaning it
+          shouldn't attempt to resize from that edge. It can for example mean
+          it's tiled next to a monitor edge on the constrained side of the
+          window.
+	</description>
+      </entry>
+      <entry name="constrained_bottom" value="13" since="7">
+	<description summary="the surface’s bottom edge is tiled">
+          The bottom edge of the window is currently constrained, meaning it
+          shouldn't attempt to resize from that edge. It can for example mean
+          it's tiled next to a monitor edge on the constrained side of the
+          window.
+	</description>
+      </entry>
     </enum>
 
     <request name="set_max_size">
@@ -908,8 +955,7 @@
 	The width and height arguments are in window geometry coordinates.
 	See xdg_surface.set_window_geometry.
 
-	Values set in this way are double-buffered. They will get applied
-	on the next commit.
+	Values set in this way are double-buffered, see wl_surface.commit.
 
 	The compositor can use this information to allow or disallow
 	different states like maximize or fullscreen and draw accurate
@@ -949,8 +995,7 @@
 	The width and height arguments are in window geometry coordinates.
 	See xdg_surface.set_window_geometry.
 
-	Values set in this way are double-buffered. They will get applied
-	on the next commit.
+	Values set in this way are double-buffered, see wl_surface.commit.
 
 	The compositor can use this information to allow or disallow
 	different states like maximize or fullscreen and draw accurate
@@ -1194,7 +1239,7 @@
     </event>
   </interface>
 
-  <interface name="xdg_popup" version="6">
+  <interface name="xdg_popup" version="7">
     <description summary="short-lived, popup surfaces for menus">
       A popup surface is a short-lived, temporary surface. It can be used to
       implement for example menus, popovers, tooltips and other similar user
