[Win32] Use WS_EX_LAYERED style for native control holders with winui composition

src/Windows/Avalonia.Win32/Interop/UnmanagedMethods.cs

@@ -1467,6 +1467,16 @@ public static extern IntPtr CreateFileMapping(IntPtr hFile,
 
         [DllImport("dwmapi.dll")]
         public static extern void DwmEnableBlurBehindWindow(IntPtr hwnd, ref DWM_BLURBEHIND blurBehind);
+
+        [Flags]
+        public enum LayeredWindowFlags
+        {
+            LWA_ALPHA = 0x00000002,
+            LWA_COLORKEY = 0x00000001,
+        }
+
+        [DllImport("user32.dll")]
+        public static extern bool SetLayeredWindowAttributes(IntPtr hwnd, uint crKey, byte bAlpha, LayeredWindowFlags dwFlags);
 
         [Flags]
         public enum DWM_BB

src/Windows/Avalonia.Win32/Win32NativeControlHost.cs

@@ -9,10 +9,12 @@ namespace Avalonia.Win32
 {
     class Win32NativeControlHost : INativeControlHostImpl
     {
+        private readonly bool _useLayeredWindow;
         public WindowImpl Window { get; }
 
-        public Win32NativeControlHost(WindowImpl window)
+        public Win32NativeControlHost(WindowImpl window, bool useLayeredWindow)
         {
+            _useLayeredWindow = useLayeredWindow;
             Window = window;
         }
 
@@ -25,12 +27,12 @@ void AssertCompatible(IPlatformHandle handle)
         public INativeControlHostDestroyableControlHandle CreateDefaultChild(IPlatformHandle parent)
         {
             AssertCompatible(parent);
-            return new DumbWindow(parent.Handle);
+            return new DumbWindow(false, parent.Handle);
         }
 
         public INativeControlHostControlTopLevelAttachment CreateNewAttachment(Func<IPlatformHandle, IPlatformHandle> create)
         {
-            var holder = new DumbWindow(Window.Handle.Handle);
+            var holder = new DumbWindow(_useLayeredWindow, Window.Handle.Handle);
             Win32NativeControlAttachment attachment = null;
             try
             {
@@ -52,7 +54,7 @@ public INativeControlHostControlTopLevelAttachment CreateNewAttachment(Func<IPla
         public INativeControlHostControlTopLevelAttachment CreateNewAttachment(IPlatformHandle handle)
         {
             AssertCompatible(handle);
-            return new Win32NativeControlAttachment(new DumbWindow(Window.Handle.Handle),
+            return new Win32NativeControlAttachment(new DumbWindow(_useLayeredWindow, Window.Handle.Handle),
                 handle) { AttachedTo = this };
         }
 
@@ -67,7 +69,7 @@ class DumbWindow : IDisposable, INativeControlHostDestroyableControlHandle
             UnmanagedMethods.WndProc _wndProcDelegate;
             private readonly string _className;
 
-            public DumbWindow(IntPtr? parent = null)
+            public DumbWindow(bool layered = false, IntPtr? parent = null)
             {
                 _wndProcDelegate = WndProc;
                 var wndClassEx = new UnmanagedMethods.WNDCLASSEX
@@ -80,7 +82,7 @@ public DumbWindow(IntPtr? parent = null)
 
                 var atom = UnmanagedMethods.RegisterClassEx(ref wndClassEx);
                 Handle = UnmanagedMethods.CreateWindowEx(
-                    0,
+                    layered ? (int)UnmanagedMethods.WindowStyles.WS_EX_LAYERED : 0,
                     atom,
                     null,
                     (int)UnmanagedMethods.WindowStyles.WS_CHILD,
@@ -92,6 +94,9 @@ public DumbWindow(IntPtr? parent = null)
                     IntPtr.Zero,
                     IntPtr.Zero,
                     IntPtr.Zero);
+                if (layered)
+                    UnmanagedMethods.SetLayeredWindowAttributes(Handle, 0, 255,
+                        UnmanagedMethods.LayeredWindowFlags.LWA_ALPHA);
             }
 
 

src/Windows/Avalonia.Win32/WindowImpl.cs

@@ -145,7 +145,7 @@ egl.Display is AngleWin32EglDisplay angleDisplay &&
 
             Screen = new ScreenImpl();
 
-            _nativeControlHost = new Win32NativeControlHost(this);
+            _nativeControlHost = new Win32NativeControlHost(this, _isUsingComposition);
             s_instances.Add(this);
         }