IGNITE-21478 Fix OOM crash on unstable topology (#11283)

Author: yurinaryshkin

modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java

@@ -575,7 +575,7 @@ public void clientEventTopologyChange(DiscoveryEvent evt, AffinityTopologyVersio
 
         Map.Entry<AffinityTopologyVersion, HistoryAffinityAssignment> prevHistEntry = affCache.floorEntry(prevVer);
 
-        HistoryAffinityAssignment newHistEntry = (prevHistEntry == null) ?
+        HistoryAffinityAssignment newHistEntry = (prevHistEntry == null || shouldCleanupShallows()) ?
             new HistoryAffinityAssignmentImpl(assignmentCpy, backups) :
             new HistoryAffinityAssignmentShallowCopy(prevHistEntry.getValue().origin(), topVer);
 
@@ -1019,6 +1019,11 @@ private boolean shouldContinueCleanup(int nonShallowSize, int totalSize) {
         return nonShallowSize > MAX_NON_SHALLOW_HIST_SIZE || totalSize > MAX_TOTAL_HIST_SIZE;
     }
 
+    /** */
+    private boolean shouldCleanupShallows() {
+        return nonShallowHistSize.get() <= MIN_NON_SHALLOW_HIST_SIZE && affCache.size() > MAX_TOTAL_HIST_SIZE;
+    }
+
     /**
      * @return All initialized versions.
      */

modules/core/src/test/java/org/apache/ignite/internal/processors/affinity/GridHistoryAffinityAssignmentTest.java

@@ -129,4 +129,32 @@ private GridAffinityAssignmentCache affinityCache(IgniteEx ignite) {
 
         return GridTestUtils.getFieldValue(cctx.affinity(), "aff");
     }
+
+    /** */
+    @Test
+    public void testAffinityCacheSizeOnCacheRecreate() throws Exception {
+        try (IgniteEx server = startGrid(0)) {
+            try (IgniteEx client = startClientGrid()) {
+                for (int i = 0; i < 300; i++) {
+                    client.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+                    server.context().cache().cacheGroups().forEach(x -> assertTrue(x.affinity().cachedVersions().size() < 260));
+
+                    client.destroyCache(DEFAULT_CACHE_NAME);
+                }
+            }
+        }
+    }
+
+    /** */
+    @Test
+    public void testAffinityCacheSizeOnReconnect() throws Exception {
+        try (IgniteEx server = startGrid(0)) {
+            for (int i = 0; i < 300; i++) {
+                try (IgniteEx client = startClientGrid()) {
+                    server.context().cache().cacheGroups().forEach(x -> assertTrue(x.affinity().cachedVersions().size() < 260));
+                }
+            }
+        }
+    }
 }