Fix OverflowException in WindowChromeWorker._HandleNCHitTest when dragging across different-DPI monitors

QuickLook.Common/Helpers/WindowHelper.cs

@@ -88,7 +88,35 @@ public static void MoveWindow(this Window window,
             out var pxWidth, out var pxHeight);
 
         // Use absolute location and relative size. WPF will scale the size to the target display
-        User32.MoveWindow(handle, (int)Math.Round(pxLeft), (int)Math.Round(pxTop), pxWidth, pxHeight, true);
+        //
+        // Guard against arithmetic overflow in WindowChromeWorker.HandleNCHitTest (net462).
+        // When the window is on a per-monitor DPI display the values here are physical pixels
+        // that may be NaN or outside the int32 range (e.g. a window straddling a negative-
+        // coordinate monitor). Feeding such a rect to User32/WPF lets the (int) casts inside
+        // Win32.MoveWindow and WindowChrome hit-testing throw OverflowException, so clamp them.
+        var x = ToInt32Clamped(pxLeft);
+        var y = ToInt32Clamped(pxTop);
+        // Keep the physical window rect strictly larger than the invisible resize border and
+        // caption. If it ever shrinks to zero/smaller, WindowChromeWorker._HandleNCHitTest
+        // (net462) computes a degenerate rect that throws OverflowException on WM_NCHITTEST.
+        var w = Math.Max(ToInt32Clamped(pxWidth), 24);
+        var h = Math.Max(ToInt32Clamped(pxHeight), 56);
+
+        User32.MoveWindow(handle, x, y, w, h, true);
+    }
+
+    private static int ToInt32Clamped(double value)
+    {
+        // Math.Round on NaN returns NaN; (int)NaN in a checked context throws OverflowException.
+        if (double.IsNaN(value))
+            return 0;
+
+        if (value <= int.MinValue)
+            return int.MinValue;
+        if (value >= int.MaxValue)
+            return int.MaxValue;
+
+        return (int)Math.Round(value);
     }
 
     public static Rect GetWindowRectInPixel(this Window window)
@@ -116,8 +144,8 @@ private static void TransformToPixels(this Visual visual,
                 matrix = src.CompositionTarget.TransformToDevice;
             }
 
-        pixelX = (int)Math.Round(matrix.M11 * unitX);
-        pixelY = (int)Math.Round(matrix.M22 * unitY);
+        pixelX = ToInt32Clamped(matrix.M11 * unitX);
+        pixelY = ToInt32Clamped(matrix.M22 * unitY);
     }
 
     public static bool IsForegroundWindowBelongToSelf()

QuickLook/App.xaml.cs

@@ -60,13 +60,21 @@ static App()
             RenderOptions.ProcessRenderMode = RenderMode.SoftwareOnly;
         }
 
-        // Explicitly set to PerMonitor to avoid being overridden by the system
-        if (SHCore.SetProcessDpiAwareness(SHCore.PROCESS_DPI_AWARENESS.PROCESS_PER_MONITOR_DPI_AWARE) is uint result)
+        // Per-Monitor V2 so a window dragged across monitors with different DPI gets WM_DPICHANGED
+        // and is rescaled automatically. This keeps WPF's per-window DPI in sync and prevents
+        // WindowChromeWorker._HandleNCHitTest from overflowing on a non-primary 4K display.
+        // Fall back to V1 (SetProcessDpiAwareness) on systems that don't support the context API.
+        if (Environment.OSVersion.Version >= new Version(10, 0, 15063) &&
+            SHCore.SetProcessDpiAwarenessContext(SHCore.DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2))
+        {
+            Debug.WriteLine("DPI Awareness context: Per-Monitor V2 applied");
+        }
+        else if (SHCore.SetProcessDpiAwareness(SHCore.PROCESS_DPI_AWARENESS.PROCESS_PER_MONITOR_DPI_AWARE) is uint result)
         {
             Debug.WriteLine(
                 result == 0 ?
-                "DPI Awareness applied successfully" :
-                $"DPI Awareness manual setup failed. Error Code: {result}"
+                "DPI Awareness (V1) applied successfully" :
+                $"DPI Awareness (V1) manual setup failed. Error Code: {result}"
             );
         }
 

QuickLook/NativeMethods/SHCore.cs

@@ -15,6 +15,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+using System;
 using System.Runtime.InteropServices;
 
 namespace QuickLook.NativeMethods;
@@ -30,4 +31,15 @@ public enum PROCESS_DPI_AWARENESS
 
     [DllImport("shcore.dll")]
     public static extern uint SetProcessDpiAwareness(PROCESS_DPI_AWARENESS awareness);
+
+    /// <summary>
+    /// DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2. Unlike the V1 context (set via
+    /// SetProcessDpiAwareness), V2 makes Windows raise WM_DPICHANGED and rescale a window as it is
+    /// dragged across monitors with different DPI. This keeps WPF's per-window DPI in sync, which
+    /// prevents WindowChromeWorker._HandleNCHitTest from overflowing on a non-primary 4K display.
+    /// </summary>
+    public static readonly nint DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = new IntPtr(-4);
+
+    [DllImport("user32.dll")]
+    public static extern bool SetProcessDpiAwarenessContext(nint value);
 }

QuickLook/ViewerWindow.Actions.cs

@@ -161,13 +161,28 @@ private void PositionWindow(Size size)
         if (WindowState == WindowState.Maximized)
             return;
 
-        size = new Size(Math.Max(MinWidth, size.Width), Math.Max(MinHeight, size.Height));
+        // Math.Max(MinWidth, NaN) keeps NaN, which then flows into the WPF window geometry and
+        // causes an OverflowException inside WindowChromeWorker.HandleNCHitTest (net462).
+        // Sanitize any NaN / non-finite / non-positive size before it reaches the window.
+        size = new Size(
+            FinitePositive(Math.Max(MinWidth, size.Width), MinWidth),
+            FinitePositive(Math.Max(MinHeight, size.Height), MinHeight));
 
         var newRect = IsLoaded ? ResizeAndCentreExistingWindow(size) : ResizeAndCentreNewWindow(size);
 
+        // MoveWindow clamps any non-finite / out-of-range coordinate and guarantees a sane
+        // physical window size, so the window can sit on a negative-coordinate monitor (e.g. a
+        // secondary display left of the primary) without being dragged to the primary screen.
         this.MoveWindow(newRect.Left, newRect.Top, newRect.Width, newRect.Height);
     }
 
+    private static double FinitePositive(double value, double fallback)
+    {
+        if (double.IsNaN(value) || double.IsInfinity(value) || value <= 0)
+            return fallback;
+        return value;
+    }
+
     private Rect ResizeAndCentreExistingWindow(Size size)
     {
         // Align window just like in macOS ...

QuickLook/ViewerWindow.xaml.cs

@@ -22,8 +22,10 @@
 using System;
 using System.Diagnostics;
 using System.IO;
+using System.Runtime.InteropServices;
 using System.Windows;
 using System.Windows.Input;
+using System.Windows.Interop;
 using System.Windows.Media;
 using System.Windows.Media.Animation;
 using System.Windows.Shell;
@@ -45,6 +47,7 @@ public partial class ViewerWindow : Window
     private string _path = string.Empty;
     private FileSystemWatcher _autoReloadWatcher;
     private readonly bool _autoReload;
+    private HwndSource _windowHwndSource;
 
     internal ViewerWindow()
     {
@@ -67,6 +70,11 @@ internal ViewerWindow()
 
         windowFrameContainer.PreviewMouseMove += ShowWindowCaptionContainer;
 
+        // Window dragging is done by hand here (WM_NCLBUTTONDOWN + HT CAPTION) instead of relying on
+        // WindowChrome's hit test, because we answer WM_NCHITTEST with HTCLIENT to prevent a net462
+        // WindowChrome overflow when dragging across different-DPI monitors.
+        titleArea.MouseLeftButtonDown += TitleArea_MouseLeftButtonDown;
+
         Topmost = SettingHelper.Get("Topmost", false);
         buttonTop.Tag = Topmost ? "Top" : "Auto";
 
@@ -188,6 +196,73 @@ protected override void OnSourceInitialized(EventArgs e)
         WindowHelper.RemoveWindowControls(this);
 
         ApplyWindowBackgroundEffects();
+
+        // Handle WM_DPICHANGED so dragging the window onto a monitor with a different DPI
+        // (e.g. a 4K secondary display on a per-monitor-DPI system) keeps the HWND geometry and
+        // WPF's view of it in sync. Without this, WindowChromeWorker._HandleNCHitTest (net462)
+        // reads a stale window rect during the drag and throws OverflowException.
+        var handle = new WindowInteropHelper(this).Handle;
+        if (handle != IntPtr.Zero && HwndSource.FromHwnd(handle) is HwndSource hwndSource)
+        {
+            _windowHwndSource = hwndSource;
+            hwndSource.AddHook(WndProc);
+        }
+    }
+
+    private IntPtr WndProc(IntPtr hwnd, int msg, IntPtr wParam, IntPtr lParam, ref bool handled)
+    {
+        const int WM_NCHITTEST = 0x0084;
+        const int WM_DPICHANGED = 0x02E0;
+
+        // Implement the hit test ourselves instead of deferring to WindowChromeWorker.
+        // WindowChromeWorker._HandleNCHitTest (net462) overflows during cross-DPI dragging (its
+        // per-window DPI math goes out of sync with the HWND rect and can't be corrected from the
+        // outside). Reporting the resize border zones here keeps edge/corner resizing working while
+        // everything else is treated as client area. Title-bar dragging is handled by
+        // TitleArea_MouseLeftButtonDown, and caption buttons are WPF content, so neither needs a
+        // non-client hit-test result.
+        if (msg == WM_NCHITTEST)
+        {
+            // Mouse position (screen, physical pixels) is packed into lParam as signed 16-bit pairs.
+            int v = lParam.ToInt32();
+            int mx = (short)(v & 0xFFFF);
+            int my = (short)((v >> 16) & 0xFFFF);
+
+            QuickLook.Common.NativeMethods.User32.GetWindowRect(hwnd, out var r);
+
+            // Resize border zone (physical pixels). The WindowChrome resize border is 6 logical
+            // pixels; a fixed 8 physical-pixel zone covers it across common DPI scale factors.
+            const int border = 8;
+            bool left = mx < r.Left + border;
+            bool right = mx >= r.Right - border;
+            bool top = my < r.Top + border;
+            bool bottom = my >= r.Bottom - border;
+
+            handled = true;
+            if (top && left) return new IntPtr(13);      // HTTOPLEFT
+            if (top && right) return new IntPtr(14);     // HTTOPRIGHT
+            if (bottom && left) return new IntPtr(16);   // HTBOTTOMLEFT
+            if (bottom && right) return new IntPtr(17);  // HTBOTTOMRIGHT
+            if (left) return new IntPtr(10);             // HTLEFT
+            if (right) return new IntPtr(11);            // HTRIGHT
+            if (top) return new IntPtr(12);              // HTTOP
+            if (bottom) return new IntPtr(15);           // HTBOTTOM
+            return new IntPtr(1);                        // HTCLIENT
+        }
+
+        if (msg == WM_DPICHANGED && lParam != IntPtr.Zero)
+        {
+            var suggested = Marshal.PtrToStructure<QuickLook.Common.NativeMethods.User32.RECT>(lParam);
+            var width = Math.Max(suggested.Right - suggested.Left, 1);
+            var height = Math.Max(suggested.Bottom - suggested.Top, 1);
+
+            QuickLook.Common.NativeMethods.User32.MoveWindow(hwnd, suggested.Left, suggested.Top, width, height, true);
+
+            handled = true;
+            return IntPtr.Zero;
+        }
+
+        return IntPtr.Zero;
     }
 
     protected override void OnContentRendered(EventArgs e)
@@ -464,6 +539,34 @@ private void ShowWindowCaptionContainer(object sender, MouseEventArgs e)
             show.Begin();
     }
 
+    private void TitleArea_MouseLeftButtonDown(object sender, MouseButtonEventArgs e)
+    {
+        if (e.LeftButton != MouseButtonState.Pressed)
+            return;
+
+        // Do not allow dragging when window is borderless (e.g. fullscreen)
+        if (WindowStyle == WindowStyle.None)
+            return;
+
+        // Start the native move loop directly. Window.DragMove() depends on a hit-test result, but
+        // we answer WM_NCHITTEST with HTCLIENT (to avoid a WindowChrome overflow), so drag by hand.
+        var hwnd = new WindowInteropHelper(this).Handle;
+        if (hwnd == IntPtr.Zero)
+            return;
+
+        ReleaseCapture();
+        SendMessage(hwnd, WM_NCLBUTTONDOWN, new IntPtr(HTCAPTION), IntPtr.Zero);
+    }
+
+    private const int WM_NCLBUTTONDOWN = 0x00A1;
+    private const int HTCAPTION = 0x0002;
+
+    [DllImport("user32.dll")]
+    private static extern bool ReleaseCapture();
+
+    [DllImport("user32.dll", CharSet = CharSet.Auto)]
+    private static extern IntPtr SendMessage(IntPtr hWnd, int msg, IntPtr wParam, IntPtr lParam);
+
     private void AutoHideCaptionContainer(object sender, EventArgs e)
     {
         if (!ContextObject.TitlebarAutoHide)