diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d36cf30f0def..54b2c7c17e14 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -179,6 +179,8 @@ Bug Fixes
 
 * GITHUB#14215: Fix degenerate case in HNSW where all vectors have the same score. (Ben Trent)
 
+* GITHUB#14320: Fix DirectIOIndexInput seek to not read when position is within buffer (Chris Hegarty)
+
 Changes in Runtime Behavior
 ---------------------
 * GITHUB#14189: Bump floor segment size to 16MB in TieredMergePolicy and
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/store/DirectIODirectory.java b/lucene/misc/src/java/org/apache/lucene/misc/store/DirectIODirectory.java
index b56a206e60b1..7eb1d625b899 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/store/DirectIODirectory.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/store/DirectIODirectory.java
@@ -373,7 +373,13 @@ public long getFilePointer() {
     @Override
     public void seek(long pos) throws IOException {
       if (pos != getFilePointer()) {
-        seekInternal(pos);
+        final long absolutePos = pos + offset;
+        if (absolutePos >= filePos && absolutePos <= filePos + buffer.limit()) {
+          // the new position is within the existing buffer
+          buffer.position(Math.toIntExact(absolutePos - filePos));
+        } else {
+          seekInternal(pos); // do an actual seek/read
+        }
       }
       assert pos == getFilePointer();
     }
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectory.java b/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectory.java
index 9d1ac2fdac3c..6d40dcce8e3b 100644
--- a/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectory.java
+++ b/lucene/misc/src/test/org/apache/lucene/misc/store/TestDirectIODirectory.java
@@ -209,4 +209,30 @@ public void testUseDirectIODefaults() throws Exception {
               OptionalLong.of(largeSize)));
     }
   }
+
+  // Ping-pong seeks should be really fast, since the position should be within buffer.
+  // The test should complete within sub-second times, not minutes.
+  public void testSeekSmall() throws IOException {
+    Path tmpDir = createTempDir("testSeekSmall");
+    try (Directory dir = getDirectory(tmpDir)) {
+      int len = atLeast(100);
+      try (IndexOutput o = dir.createOutput("out", newIOContext(random()))) {
+        byte[] b = new byte[len];
+        for (int i = 0; i < len; i++) {
+          b[i] = (byte) i;
+        }
+        o.writeBytes(b, 0, len);
+      }
+      try (IndexInput in = dir.openInput("out", newIOContext(random()))) {
+        for (int i = 0; i < 100_000; i++) {
+          in.seek(2);
+          assertEquals(2, in.readByte());
+          in.seek(1);
+          assertEquals(1, in.readByte());
+          in.seek(0);
+          assertEquals(0, in.readByte());
+        }
+      }
+    }
+  }
 }