Detect stacks of maximized windows in the overlap offset and badge

Rectangle/StackBadge/StackBadgeGeometry.swift

@@ -70,6 +70,34 @@ enum StackBadgeGeometry {
         return best
     }
 
+    /// Indices of the cascade stack, when some candidates cover the screen.
+    ///
+    /// A window covering the screen shares its origin with every left/right
+    /// half and corner placement, so counting one would inflate the badge on
+    /// ordinary tiled stacks. Tiled windows therefore win: the stack is taken
+    /// from them whenever they form one. Only when they don't is the stack
+    /// taken from the screen-covering windows - a pile of maximized windows,
+    /// where the badge is the only way to see what's buried.
+    static func stackIndices(among origins: [CGPoint],
+                             coversScreen: [Bool],
+                             cascadeRange: CGFloat,
+                             tolerance: CGFloat) -> [Int] {
+        guard coversScreen.count == origins.count else { return [] }
+
+        func stack(of indices: [Int]) -> [Int] {
+            stackIndices(among: indices.map { origins[$0] },
+                         cascadeRange: cascadeRange,
+                         tolerance: tolerance)
+                .map { indices[$0] }
+        }
+
+        let tiled = stack(of: origins.indices.filter { !coversScreen[$0] })
+        guard tiled.count < 2 else { return tiled }
+
+        let covering = stack(of: origins.indices.filter { coversScreen[$0] })
+        return covering.count >= 2 ? covering : tiled
+    }
+
     /// The corner whose hover zone contains the point, or nil. The zone is a
     /// square extending right and down from the corner (down in AppKit means
     /// minus y), covering where gap-shifted windows and their title bars sit

Rectangle/StackBadge/StackBadgeManager.swift

@@ -151,9 +151,6 @@ class StackBadgeManager {
         // upward move in AX space), so the y window is symmetric.
         let candidates = WindowUtil.getWindowList().filter { info in
             guard info.level == kCGNormalWindowLevel else { return false }
-            let coversScreen = info.frame.width > screenFrameAX.width * 0.9
-                && info.frame.height > screenFrameAX.height * 0.9
-            guard !coversScreen else { return false }
             let dx = info.frame.origin.x - cornerAX.x
             let dy = info.frame.origin.y - cornerAX.y
             return dx >= -tolerance && dx <= candidateRange
@@ -162,9 +159,18 @@ class StackBadgeManager {
 
         // ...and the stack is the densest cascade cluster among them, so
         // the widened box doesn't count unrelated neighbors and an
-        // unrelated leftmost window doesn't mask a real stack.
+        // unrelated leftmost window doesn't mask a real stack. Windows
+        // covering the screen only form the stack when the tiled ones don't,
+        // so a pile of maximized windows is still revealed.
+        let coversScreen = candidates.map {
+            $0.frame.width > screenFrameAX.width * 0.9
+                && $0.frame.height > screenFrameAX.height * 0.9
+        }
         let stacked = StackBadgeGeometry
-            .stackIndices(among: candidates.map { $0.frame.origin }, cascadeRange: cascadeRange, tolerance: tolerance)
+            .stackIndices(among: candidates.map { $0.frame.origin },
+                          coversScreen: coversScreen,
+                          cascadeRange: cascadeRange,
+                          tolerance: tolerance)
             .map { candidates[$0] }
 
         guard stacked.count >= 2,

Rectangle/WindowManager.swift

@@ -135,7 +135,11 @@ class WindowManager {
             calcResult.rect = GapCalculation.applyGaps(calcResult.rect, dimension: gapsApplicable, sharedEdges: gapSharedEdges, gapSize: Defaults.gapSize.value, skipTopGap: Defaults.skipGapTopEdge.enabled)
         }
 
-        if Defaults.cyclingOverlapOffset.userEnabled, action.positionCycles {
+        // Maximize doesn't cycle through positions, but two maximized windows
+        // still land exactly on top of each other, which is what the offset
+        // exists to reveal. (When gaps apply there's room to shift into; with
+        // no gaps the clamping in the offset leaves the window where it was.)
+        if Defaults.cyclingOverlapOffset.userEnabled, action.positionCycles || action == .maximize {
             calcResult.rect = applyOverlapOffsetIfNeeded(calcResult.rect, windowId: windowId, screen: calcResult.screen)
         }
 
@@ -296,6 +300,15 @@ class WindowManager {
         var candidate = rect
         var cascadeLevel = 0
 
+        // A window covering the screen shares its origin with every left/right
+        // half and corner placement, so matching one would offset all of them
+        // (#1766) - it's ignored. That only holds while the window being placed
+        // is smaller than it: when this window covers the screen too, a shared
+        // origin is a genuine stack of maximized windows, which is exactly what
+        // the offset is meant to reveal.
+        let candidateCoversScreen = rect.width > screenFrameAX.width * 0.9
+            && rect.height > screenFrameAX.height * 0.9
+
         while cascadeLevel < maxCascade {
             let candidateAX = candidate.screenFlipped
             let hasOverlap = otherWindows.contains { element in
@@ -304,7 +317,7 @@ class WindowManager {
                     && abs(otherFrame.origin.y - candidateAX.origin.y) < tolerance
                 let otherCoversScreen = otherFrame.width > screenFrameAX.width * 0.9
                     && otherFrame.height > screenFrameAX.height * 0.9
-                return originsMatch && !otherCoversScreen
+                return originsMatch && (candidateCoversScreen || !otherCoversScreen)
             }
 
             guard hasOverlap else { break }

RectangleTests/RectangleTests.swift

@@ -258,6 +258,46 @@ class StackBadgeGeometryTests: XCTestCase {
         XCTAssertTrue(StackBadgeGeometry.stackIndices(among: [], cascadeRange: 15, tolerance: 4).isEmpty)
     }
 
+    // A pile of maximized windows shares one origin and is invisible without
+    // the badge, so it forms a stack when no tiled stack is present.
+    func testStackClusterFallsBackToScreenCoveringWindows() {
+        let origins = [CGPoint(x: 0, y: 0), CGPoint(x: 0, y: 0), CGPoint(x: 11, y: -11)]
+        let indices = StackBadgeGeometry.stackIndices(among: origins,
+                                                      coversScreen: [true, true, true],
+                                                      cascadeRange: 15, tolerance: 4)
+        XCTAssertEqual(indices.count, 3)
+    }
+
+    // A single maximized window sharing its origin with one tiled window is
+    // not a stack - counting it would inflate the badge on ordinary layouts.
+    func testStackClusterIgnoresLoneScreenCoveringWindow() {
+        let origins = [CGPoint(x: 0, y: 0), CGPoint(x: 0, y: 0)]
+        let indices = StackBadgeGeometry.stackIndices(among: origins,
+                                                      coversScreen: [true, false],
+                                                      cascadeRange: 15, tolerance: 4)
+        // Fewer than two is no stack: the badge requires two windows, so the
+        // maximized window must not pair up with the single tiled one.
+        XCTAssertLessThan(indices.count, 2)
+        XCTAssertFalse(indices.contains(0))
+    }
+
+    // Tiled windows win: a maximized window sharing the corner must not be
+    // added to a stack of tiled windows.
+    func testStackClusterPrefersTiledOverScreenCovering() {
+        let origins = [CGPoint(x: 0, y: 0), CGPoint(x: 0, y: 0), CGPoint(x: 11, y: -11)]
+        let indices = StackBadgeGeometry.stackIndices(among: origins,
+                                                      coversScreen: [true, false, false],
+                                                      cascadeRange: 15, tolerance: 4)
+        XCTAssertEqual(indices.sorted(), [1, 2])
+    }
+
+    func testStackClusterMismatchedCoveringFlagsIsEmpty() {
+        let indices = StackBadgeGeometry.stackIndices(among: [CGPoint(x: 0, y: 0)],
+                                                      coversScreen: [],
+                                                      cascadeRange: 15, tolerance: 4)
+        XCTAssertTrue(indices.isEmpty)
+    }
+
     // Regression (review finding): an unrelated window that happens to be the
     // leftmost candidate in the gap-widened box must not mask the real stack.
     func testStackClusterLeftOutlierDoesNotMaskStack() {