Skip to content

Commit 66de7ba

Browse files
lorbanlorban
authored
Improve ssl buffers handling (#8165)
* Fixes #8161 improve SSLConnection buffers handling Added memory heuristic to ArrayRetainableByteBufferPool Signed-off-by: Ludovic Orban <[email protected]>
1 parent 0699bc5 commit 66de7ba

File tree

5 files changed

+457
-55
lines changed

5 files changed

+457
-55
lines changed

‎jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java

Lines changed: 310 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@
2222
import java.net.SocketTimeoutException;
2323
import java.nio.ByteBuffer;
2424
import java.nio.charset.StandardCharsets;
25+
import java.util.ArrayList;
2526
import java.util.Arrays;
2627
import java.util.List;
2728
import java.util.concurrent.CountDownLatch;
2829
import java.util.concurrent.ExecutionException;
2930
import java.util.concurrent.Executor;
3031
import java.util.concurrent.TimeUnit;
32+
import java.util.concurrent.atomic.AtomicBoolean;
3133
import java.util.concurrent.atomic.AtomicLong;
3234
import java.util.concurrent.atomic.AtomicReference;
3335
import javax.net.ssl.SSLEngine;
@@ -36,31 +38,39 @@
3638
import javax.net.ssl.SSLHandshakeException;
3739
import javax.net.ssl.SSLPeerUnverifiedException;
3840
import javax.net.ssl.SSLSocket;
41+
import javax.servlet.http.HttpServletRequest;
42+
import javax.servlet.http.HttpServletResponse;
3943

4044
import org.eclipse.jetty.client.api.ContentResponse;
4145
import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
4246
import org.eclipse.jetty.http.HttpHeader;
4347
import org.eclipse.jetty.http.HttpHeaderValue;
4448
import org.eclipse.jetty.http.HttpScheme;
4549
import org.eclipse.jetty.http.HttpStatus;
50+
import org.eclipse.jetty.io.ArrayByteBufferPool;
51+
import org.eclipse.jetty.io.ArrayRetainableByteBufferPool;
4652
import org.eclipse.jetty.io.ByteBufferPool;
4753
import org.eclipse.jetty.io.ClientConnectionFactory;
4854
import org.eclipse.jetty.io.ClientConnector;
4955
import org.eclipse.jetty.io.Connection;
5056
import org.eclipse.jetty.io.ConnectionStatistics;
5157
import org.eclipse.jetty.io.EndPoint;
58+
import org.eclipse.jetty.io.RetainableByteBuffer;
59+
import org.eclipse.jetty.io.RetainableByteBufferPool;
5260
import org.eclipse.jetty.io.ssl.SslClientConnectionFactory;
5361
import org.eclipse.jetty.io.ssl.SslConnection;
5462
import org.eclipse.jetty.io.ssl.SslHandshakeListener;
5563
import org.eclipse.jetty.server.Connector;
5664
import org.eclipse.jetty.server.Handler;
5765
import org.eclipse.jetty.server.HttpConfiguration;
5866
import org.eclipse.jetty.server.HttpConnectionFactory;
67+
import org.eclipse.jetty.server.Request;
5968
import org.eclipse.jetty.server.SecureRequestCustomizer;
6069
import org.eclipse.jetty.server.Server;
6170
import org.eclipse.jetty.server.ServerConnector;
6271
import org.eclipse.jetty.server.SslConnectionFactory;
6372
import org.eclipse.jetty.toolchain.test.Net;
73+
import org.eclipse.jetty.util.Pool;
6474
import org.eclipse.jetty.util.StringUtil;
6575
import org.eclipse.jetty.util.ssl.SslContextFactory;
6676
import org.eclipse.jetty.util.thread.ExecutorThreadPool;
@@ -71,9 +81,14 @@
7181
import org.junit.jupiter.api.Test;
7282
import org.junit.jupiter.api.condition.EnabledForJreRange;
7383
import org.junit.jupiter.api.condition.JRE;
84+
import org.junit.jupiter.params.ParameterizedTest;
85+
import org.junit.jupiter.params.provider.ValueSource;
7486

87+
import static org.awaitility.Awaitility.await;
7588
import static org.hamcrest.MatcherAssert.assertThat;
89+
import static org.hamcrest.Matchers.empty;
7690
import static org.hamcrest.Matchers.instanceOf;
91+
import static org.hamcrest.Matchers.is;
7792
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
7893
import static org.junit.jupiter.api.Assertions.assertEquals;
7994
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -682,12 +697,7 @@ protected int networkFill(ByteBuffer input) throws IOException
682697
// Trigger the creation of a new connection, but don't use it.
683698
ConnectionPoolHelper.tryCreate(connectionPool);
684699
// Verify that the connection has been created.
685-
while (true)
686-
{
687-
Thread.sleep(50);
688-
if (connectionPool.getConnectionCount() == 1)
689-
break;
690-
}
700+
await().atMost(5, TimeUnit.SECONDS).until(connectionPool::getConnectionCount, is(1));
691701

692702
// Wait for the server to idle timeout the connection.
693703
Thread.sleep(idleTimeout + idleTimeout / 2);
@@ -698,6 +708,299 @@ protected int networkFill(ByteBuffer input) throws IOException
698708
assertEquals(0, clientBytes.get());
699709
}
700710

711+
@Test
712+
public void testEncryptedInputBufferRepooling() throws Exception
713+
{
714+
SslContextFactory.Server serverTLSFactory = createServerSslContextFactory();
715+
QueuedThreadPool serverThreads = new QueuedThreadPool();
716+
serverThreads.setName("server");
717+
server = new Server(serverThreads);
718+
var retainableByteBufferPool = new ArrayRetainableByteBufferPool()
719+
{
720+
@Override
721+
public Pool<RetainableByteBuffer> poolFor(int capacity, boolean direct)
722+
{
723+
return super.poolFor(capacity, direct);
724+
}
725+
};
726+
server.addBean(retainableByteBufferPool);
727+
HttpConfiguration httpConfig = new HttpConfiguration();
728+
httpConfig.addCustomizer(new SecureRequestCustomizer());
729+
HttpConnectionFactory http = new HttpConnectionFactory(httpConfig);
730+
SslConnectionFactory ssl = new SslConnectionFactory(serverTLSFactory, http.getProtocol())
731+
{
732+
@Override
733+
protected SslConnection newSslConnection(Connector connector, EndPoint endPoint, SSLEngine engine)
734+
{
735+
ByteBufferPool byteBufferPool = connector.getByteBufferPool();
736+
RetainableByteBufferPool retainableByteBufferPool = connector.getBean(RetainableByteBufferPool.class);
737+
return new SslConnection(retainableByteBufferPool, byteBufferPool, connector.getExecutor(), endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
738+
{
739+
@Override
740+
protected int networkFill(ByteBuffer input) throws IOException
741+
{
742+
int n = super.networkFill(input);
743+
if (n > 0)
744+
throw new IOException("boom");
745+
return n;
746+
}
747+
};
748+
}
749+
};
750+
connector = new ServerConnector(server, 1, 1, ssl, http);
751+
server.addConnector(connector);
752+
server.setHandler(new EmptyServerHandler());
753+
server.start();
754+
755+
SslContextFactory.Client clientTLSFactory = createClientSslContextFactory();
756+
ClientConnector clientConnector = new ClientConnector();
757+
clientConnector.setSelectors(1);
758+
clientConnector.setSslContextFactory(clientTLSFactory);
759+
QueuedThreadPool clientThreads = new QueuedThreadPool();
760+
clientThreads.setName("client");
761+
clientConnector.setExecutor(clientThreads);
762+
client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector));
763+
client.setExecutor(clientThreads);
764+
client.start();
765+
766+
assertThrows(Exception.class, () -> client.newRequest("localhost", connector.getLocalPort()).scheme(HttpScheme.HTTPS.asString()).send());
767+
768+
Pool<RetainableByteBuffer> bucket = retainableByteBufferPool.poolFor(16 * 1024 + 1, ssl.isDirectBuffersForEncryption());
769+
assertEquals(1, bucket.size());
770+
assertEquals(1, bucket.getIdleCount());
771+
}
772+
773+
@Test
774+
public void testEncryptedOutputBufferRepooling() throws Exception
775+
{
776+
SslContextFactory.Server serverTLSFactory = createServerSslContextFactory();
777+
QueuedThreadPool serverThreads = new QueuedThreadPool();
778+
serverThreads.setName("server");
779+
server = new Server(serverThreads);
780+
List<ByteBuffer> leakedBuffers = new ArrayList<>();
781+
ArrayByteBufferPool byteBufferPool = new ArrayByteBufferPool()
782+
{
783+
@Override
784+
public ByteBuffer acquire(int size, boolean direct)
785+
{
786+
ByteBuffer acquired = super.acquire(size, direct);
787+
leakedBuffers.add(acquired);
788+
return acquired;
789+
}
790+
791+
@Override
792+
public void release(ByteBuffer buffer)
793+
{
794+
leakedBuffers.remove(buffer);
795+
super.release(buffer);
796+
}
797+
};
798+
server.addBean(byteBufferPool);
799+
HttpConfiguration httpConfig = new HttpConfiguration();
800+
httpConfig.addCustomizer(new SecureRequestCustomizer());
801+
HttpConnectionFactory http = new HttpConnectionFactory(httpConfig);
802+
SslConnectionFactory ssl = new SslConnectionFactory(serverTLSFactory, http.getProtocol())
803+
{
804+
@Override
805+
protected SslConnection newSslConnection(Connector connector, EndPoint endPoint, SSLEngine engine)
806+
{
807+
ByteBufferPool byteBufferPool = connector.getByteBufferPool();
808+
RetainableByteBufferPool retainableByteBufferPool = connector.getBean(RetainableByteBufferPool.class);
809+
return new SslConnection(retainableByteBufferPool, byteBufferPool, connector.getExecutor(), endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
810+
{
811+
@Override
812+
protected boolean networkFlush(ByteBuffer output) throws IOException
813+
{
814+
throw new IOException("bang");
815+
}
816+
};
817+
}
818+
};
819+
connector = new ServerConnector(server, 1, 1, ssl, http);
820+
server.addConnector(connector);
821+
server.setHandler(new EmptyServerHandler());
822+
server.start();
823+
824+
SslContextFactory.Client clientTLSFactory = createClientSslContextFactory();
825+
ClientConnector clientConnector = new ClientConnector();
826+
clientConnector.setSelectors(1);
827+
clientConnector.setSslContextFactory(clientTLSFactory);
828+
QueuedThreadPool clientThreads = new QueuedThreadPool();
829+
clientThreads.setName("client");
830+
clientConnector.setExecutor(clientThreads);
831+
client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector));
832+
client.setExecutor(clientThreads);
833+
client.start();
834+
835+
assertThrows(Exception.class, () -> client.newRequest("localhost", connector.getLocalPort()).scheme(HttpScheme.HTTPS.asString()).send());
836+
837+
await().atMost(5, TimeUnit.SECONDS).until(() -> leakedBuffers, is(empty()));
838+
}
839+
840+
@ParameterizedTest
841+
@ValueSource(booleans = {true, false})
842+
public void testEncryptedOutputBufferRepoolingAfterNetworkFlushReturnsFalse(boolean close) throws Exception
843+
{
844+
SslContextFactory.Server serverTLSFactory = createServerSslContextFactory();
845+
QueuedThreadPool serverThreads = new QueuedThreadPool();
846+
serverThreads.setName("server");
847+
server = new Server(serverThreads);
848+
List<ByteBuffer> leakedBuffers = new ArrayList<>();
849+
ArrayByteBufferPool byteBufferPool = new ArrayByteBufferPool()
850+
{
851+
@Override
852+
public ByteBuffer acquire(int size, boolean direct)
853+
{
854+
ByteBuffer acquired = super.acquire(size, direct);
855+
leakedBuffers.add(acquired);
856+
return acquired;
857+
}
858+
859+
@Override
860+
public void release(ByteBuffer buffer)
861+
{
862+
leakedBuffers.remove(buffer);
863+
super.release(buffer);
864+
}
865+
};
866+
server.addBean(byteBufferPool);
867+
HttpConfiguration httpConfig = new HttpConfiguration();
868+
httpConfig.addCustomizer(new SecureRequestCustomizer());
869+
HttpConnectionFactory http = new HttpConnectionFactory(httpConfig);
870+
AtomicBoolean failFlush = new AtomicBoolean(false);
871+
SslConnectionFactory ssl = new SslConnectionFactory(serverTLSFactory, http.getProtocol())
872+
{
873+
@Override
874+
protected SslConnection newSslConnection(Connector connector, EndPoint endPoint, SSLEngine engine)
875+
{
876+
ByteBufferPool byteBufferPool = connector.getByteBufferPool();
877+
RetainableByteBufferPool retainableByteBufferPool = connector.getBean(RetainableByteBufferPool.class);
878+
return new SslConnection(retainableByteBufferPool, byteBufferPool, connector.getExecutor(), endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
879+
{
880+
@Override
881+
protected boolean networkFlush(ByteBuffer output) throws IOException
882+
{
883+
if (failFlush.get())
884+
return false;
885+
return super.networkFlush(output);
886+
}
887+
};
888+
}
889+
};
890+
connector = new ServerConnector(server, 1, 1, ssl, http);
891+
server.addConnector(connector);
892+
server.setHandler(new EmptyServerHandler()
893+
{
894+
@Override
895+
protected void service(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response)
896+
{
897+
failFlush.set(true);
898+
if (close)
899+
jettyRequest.getHttpChannel().getEndPoint().close();
900+
else
901+
jettyRequest.getHttpChannel().getEndPoint().shutdownOutput();
902+
}
903+
});
904+
server.start();
905+
906+
SslContextFactory.Client clientTLSFactory = createClientSslContextFactory();
907+
ClientConnector clientConnector = new ClientConnector();
908+
clientConnector.setSelectors(1);
909+
clientConnector.setSslContextFactory(clientTLSFactory);
910+
QueuedThreadPool clientThreads = new QueuedThreadPool();
911+
clientThreads.setName("client");
912+
clientConnector.setExecutor(clientThreads);
913+
client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector));
914+
client.setExecutor(clientThreads);
915+
client.start();
916+
917+
assertThrows(Exception.class, () -> client.newRequest("localhost", connector.getLocalPort()).scheme(HttpScheme.HTTPS.asString()).send());
918+
919+
await().atMost(5, TimeUnit.SECONDS).until(() -> leakedBuffers, is(empty()));
920+
}
921+
922+
@ParameterizedTest
923+
@ValueSource(booleans = {true, false})
924+
public void testEncryptedOutputBufferRepoolingAfterNetworkFlushThrows(boolean close) throws Exception
925+
{
926+
SslContextFactory.Server serverTLSFactory = createServerSslContextFactory();
927+
QueuedThreadPool serverThreads = new QueuedThreadPool();
928+
serverThreads.setName("server");
929+
server = new Server(serverThreads);
930+
List<ByteBuffer> leakedBuffers = new ArrayList<>();
931+
ArrayByteBufferPool byteBufferPool = new ArrayByteBufferPool()
932+
{
933+
@Override
934+
public ByteBuffer acquire(int size, boolean direct)
935+
{
936+
ByteBuffer acquired = super.acquire(size, direct);
937+
leakedBuffers.add(acquired);
938+
return acquired;
939+
}
940+
941+
@Override
942+
public void release(ByteBuffer buffer)
943+
{
944+
leakedBuffers.remove(buffer);
945+
super.release(buffer);
946+
}
947+
};
948+
server.addBean(byteBufferPool);
949+
HttpConfiguration httpConfig = new HttpConfiguration();
950+
httpConfig.addCustomizer(new SecureRequestCustomizer());
951+
HttpConnectionFactory http = new HttpConnectionFactory(httpConfig);
952+
AtomicBoolean failFlush = new AtomicBoolean(false);
953+
SslConnectionFactory ssl = new SslConnectionFactory(serverTLSFactory, http.getProtocol())
954+
{
955+
@Override
956+
protected SslConnection newSslConnection(Connector connector, EndPoint endPoint, SSLEngine engine)
957+
{
958+
ByteBufferPool byteBufferPool = connector.getByteBufferPool();
959+
RetainableByteBufferPool retainableByteBufferPool = connector.getBean(RetainableByteBufferPool.class);
960+
return new SslConnection(retainableByteBufferPool, byteBufferPool, connector.getExecutor(), endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
961+
{
962+
@Override
963+
protected boolean networkFlush(ByteBuffer output) throws IOException
964+
{
965+
if (failFlush.get())
966+
throw new IOException();
967+
return super.networkFlush(output);
968+
}
969+
};
970+
}
971+
};
972+
connector = new ServerConnector(server, 1, 1, ssl, http);
973+
server.addConnector(connector);
974+
server.setHandler(new EmptyServerHandler()
975+
{
976+
@Override
977+
protected void service(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
978+
{
979+
failFlush.set(true);
980+
if (close)
981+
jettyRequest.getHttpChannel().getEndPoint().close();
982+
else
983+
jettyRequest.getHttpChannel().getEndPoint().shutdownOutput();
984+
}
985+
});
986+
server.start();
987+
988+
SslContextFactory.Client clientTLSFactory = createClientSslContextFactory();
989+
ClientConnector clientConnector = new ClientConnector();
990+
clientConnector.setSelectors(1);
991+
clientConnector.setSslContextFactory(clientTLSFactory);
992+
QueuedThreadPool clientThreads = new QueuedThreadPool();
993+
clientThreads.setName("client");
994+
clientConnector.setExecutor(clientThreads);
995+
client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector));
996+
client.setExecutor(clientThreads);
997+
client.start();
998+
999+
assertThrows(Exception.class, () -> client.newRequest("localhost", connector.getLocalPort()).scheme(HttpScheme.HTTPS.asString()).send());
1000+
1001+
await().atMost(5, TimeUnit.SECONDS).until(() -> leakedBuffers, is(empty()));
1002+
}
1003+
7011004
@Test
7021005
public void testNeverUsedConnectionThenClientIdleTimeout() throws Exception
7031006
{
@@ -780,12 +1083,7 @@ protected int networkFill(ByteBuffer input) throws IOException
7801083
// Trigger the creation of a new connection, but don't use it.
7811084
ConnectionPoolHelper.tryCreate(connectionPool);
7821085
// Verify that the connection has been created.
783-
while (true)
784-
{
785-
Thread.sleep(50);
786-
if (connectionPool.getConnectionCount() == 1)
787-
break;
788-
}
1086+
await().atMost(5, TimeUnit.SECONDS).until(connectionPool::getConnectionCount, is(1));
7891087

7901088
// Wait for the client to idle timeout the connection.
7911089
Thread.sleep(idleTimeout + idleTimeout / 2);

0 commit comments

Comments
 (0)