[2025-04-29T18:29:29,191][WARN ][o.o.i.c.IndicesClusterStateService] [node_name_9200] [security-auditlog-2025.04.29][0] marking and sending shard failed due to [shard failure, reason [merge failed]]
org.apache.lucene.index.MergePolicy$MergeException: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot merge
	at org.opensearch.index.engine.InternalEngine$EngineMergeScheduler$2.doRun(InternalEngine.java:2534) ~[opensearch-3.0.0.jar:3.0.0]
	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:975) ~[opensearch-3.0.0.jar:3.0.0]
	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) ~[opensearch-3.0.0.jar:3.0.0]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot merge
	at org.apache.lucene.index.IndexWriter.hasPendingMerges(IndexWriter.java:2427) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.opensearch.index.engine.InternalEngine$EngineMergeScheduler.afterMerge(InternalEngine.java:2492) ~[opensearch-3.0.0.jar:3.0.0]
	at org.opensearch.index.engine.OpenSearchConcurrentMergeScheduler.doMerge(OpenSearchConcurrentMergeScheduler.java:131) ~[opensearch-3.0.0.jar:3.0.0]
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:729) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
Caused by: java.io.IOException: Operation not permitted
	at java.base/java.nio.MappedMemoryUtils.isLoaded0(Native Method) ~[?:?]
	at java.base/java.nio.MappedMemoryUtils.isLoaded(MappedMemoryUtils.java:45) ~[?:?]
	at java.base/java.nio.Buffer$2.isLoaded(Buffer.java:905) ~[?:?]
	at java.base/jdk.internal.misc.ScopedMemoryAccess.isLoadedInternal(ScopedMemoryAccess.java:254) ~[?:?]
	at java.base/jdk.internal.misc.ScopedMemoryAccess.isLoaded(ScopedMemoryAccess.java:242) ~[?:?]
	at java.base/jdk.internal.foreign.MappedMemorySegmentImpl.isLoaded(MappedMemorySegmentImpl.java:87) ~[?:?]
	at org.apache.lucene.store.MemorySegmentIndexInput.lambda$prefetch$1(MemorySegmentIndexInput.java:355) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.store.MemorySegmentIndexInput.advise(MemorySegmentIndexInput.java:414) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.store.MemorySegmentIndexInput.prefetch(MemorySegmentIndexInput.java:351) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene101.Lucene101PostingsReader.prefetchPostings(Lucene101PostingsReader.java:1180) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene101.Lucene101PostingsReader$BlockPostingsEnum.reset(Lucene101PostingsReader.java:494) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene101.Lucene101PostingsReader.postings(Lucene101PostingsReader.java:278) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.postings(SegmentTermsEnum.java:1107) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.MultiTermsEnum.postings(MultiTermsEnum.java:341) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.MappedMultiFields$MappedMultiTermsEnum.postings(MappedMultiFields.java:132) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:126) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter$TermsWriter.write(Lucene90BlockTreeTermsWriter.java:1080) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter.write(Lucene90BlockTreeTermsWriter.java:399) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:95) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:195) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:222) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.SegmentMerger.mergeWithLogging(SegmentMerger.java:314) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:146) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:5282) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4748) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6566) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:668) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.opensearch.index.engine.OpenSearchConcurrentMergeScheduler.doMerge(OpenSearchConcurrentMergeScheduler.java:126) ~[opensearch-3.0.0.jar:3.0.0]
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:729) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
[2025-04-29T18:29:29,193][WARN ][o.o.c.r.a.AllocationService] [node_name_9200] failing shard [failed shard, shard [security-auditlog-2025.04.29][0], node[oqnLVrINRZCvhxZf7iwN8w], [P], s[STARTED], a[id=29rOCLg4RX-3mNHpzkzgGw], message [shard failure, reason [merge failed]], failure [MergeException[java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot merge]; nested: IllegalStateException[this writer hit an unrecoverable error; cannot merge]; nested: IOException[Operation not permitted]; ], markAsStale [true]]
org.apache.lucene.index.MergePolicy$MergeException: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot merge
	at org.opensearch.index.engine.InternalEngine$EngineMergeScheduler$2.doRun(InternalEngine.java:2534) ~[opensearch-3.0.0.jar:3.0.0]
	at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:975) ~[opensearch-3.0.0.jar:3.0.0]
	at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) ~[opensearch-3.0.0.jar:3.0.0]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot merge
	at org.apache.lucene.index.IndexWriter.hasPendingMerges(IndexWriter.java:2427) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.opensearch.index.engine.InternalEngine$EngineMergeScheduler.afterMerge(InternalEngine.java:2492) ~[opensearch-3.0.0.jar:3.0.0]
	at org.opensearch.index.engine.OpenSearchConcurrentMergeScheduler.doMerge(OpenSearchConcurrentMergeScheduler.java:131) ~[opensearch-3.0.0.jar:3.0.0]
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:729) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
Caused by: java.io.IOException: Operation not permitted
	at java.base/java.nio.MappedMemoryUtils.isLoaded0(Native Method) ~[?:?]
	at java.base/java.nio.MappedMemoryUtils.isLoaded(MappedMemoryUtils.java:45) ~[?:?]
	at java.base/java.nio.Buffer$2.isLoaded(Buffer.java:905) ~[?:?]
	at java.base/jdk.internal.misc.ScopedMemoryAccess.isLoadedInternal(ScopedMemoryAccess.java:254) ~[?:?]
	at java.base/jdk.internal.misc.ScopedMemoryAccess.isLoaded(ScopedMemoryAccess.java:242) ~[?:?]
	at java.base/jdk.internal.foreign.MappedMemorySegmentImpl.isLoaded(MappedMemorySegmentImpl.java:87) ~[?:?]
	at org.apache.lucene.store.MemorySegmentIndexInput.lambda$prefetch$1(MemorySegmentIndexInput.java:355) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.store.MemorySegmentIndexInput.advise(MemorySegmentIndexInput.java:414) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.store.MemorySegmentIndexInput.prefetch(MemorySegmentIndexInput.java:351) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene101.Lucene101PostingsReader.prefetchPostings(Lucene101PostingsReader.java:1180) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene101.Lucene101PostingsReader$BlockPostingsEnum.reset(Lucene101PostingsReader.java:494) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene101.Lucene101PostingsReader.postings(Lucene101PostingsReader.java:278) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene90.blocktree.SegmentTermsEnum.postings(SegmentTermsEnum.java:1107) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.MultiTermsEnum.postings(MultiTermsEnum.java:341) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.MappedMultiFields$MappedMultiTermsEnum.postings(MappedMultiFields.java:132) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:126) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter$TermsWriter.write(Lucene90BlockTreeTermsWriter.java:1080) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter.write(Lucene90BlockTreeTermsWriter.java:399) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:95) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:195) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:222) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.SegmentMerger.mergeWithLogging(SegmentMerger.java:314) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:146) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:5282) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4748) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:6566) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:668) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]
	at org.opensearch.index.engine.OpenSearchConcurrentMergeScheduler.doMerge(OpenSearchConcurrentMergeScheduler.java:126) ~[opensearch-3.0.0.jar:3.0.0]
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:729) ~[lucene-core-10.1.0.jar:10.1.0 884954006de769dc43b811267230d625886e6515 - 2024-12-17 16:15:44]