Skip to content

Commit 6b58457

Browse files
steve-todorovsteve-todorov
authored
Merge pull request #855 from /issues/854/attr-cache-improvements
Improve file attribute caching
2 parents 03cd6fb + 596434b commit 6b58457

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+1345
-436
lines changed

‎build.gradle.kts

Lines changed: 52 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ plugins {
1515

1616
allprojects {
1717
repositories {
18-
mavenLocal()
18+
//mavenLocal()
1919
// Allows you to specify your own repository manager instance.
2020
if (project.hasProperty("s3fs.proxy.url")) {
2121
maven {
@@ -38,6 +38,50 @@ java {
3838
withJavadocJar()
3939
}
4040

41+
// Configure multiple test sources
42+
testing {
43+
suites {
44+
// Just for self reference, technically this is already configured by default.
45+
val test by getting(JvmTestSuite::class) {
46+
useJUnitJupiter() // already the default.
47+
testType.set(TestSuiteType.UNIT_TEST) // already the default.
48+
}
49+
50+
// testIntegration test sources
51+
val testIntegration by registering(JvmTestSuite::class) {
52+
val self = this
53+
testType.set(TestSuiteType.INTEGRATION_TEST)
54+
55+
// We need to manually add the "main" sources to the classpath.
56+
sourceSets {
57+
named(self.name) {
58+
compileClasspath += sourceSets.main.get().output + sourceSets.test.get().output
59+
runtimeClasspath += sourceSets.main.get().output + sourceSets.test.get().output
60+
}
61+
}
62+
63+
// Inherit implementation, runtime and test dependencies (adds them to the compile classpath)
64+
configurations.named("${self.name}Implementation") {
65+
extendsFrom(configurations.testImplementation.get())
66+
extendsFrom(configurations.runtimeOnly.get())
67+
extendsFrom(configurations.implementation.get())
68+
}
69+
70+
// Make sure the integration test is executed as part of the "check" task.
71+
tasks.named<Task>("check") {
72+
dependsOn(named<JvmTestSuite>(self.name))
73+
}
74+
75+
tasks.named<Task>(self.name) {
76+
mustRunAfter(test)
77+
}
78+
79+
}
80+
}
81+
82+
83+
}
84+
4185
dependencies {
4286
api(platform("software.amazon.awssdk:bom:2.29.9"))
4387
api("software.amazon.awssdk:s3") {
@@ -49,6 +93,9 @@ dependencies {
4993
exclude("org.slf4j", "slf4j-api")
5094
}
5195
api("com.google.code.findbugs:jsr305:3.0.2")
96+
api("com.github.ben-manes.caffeine:caffeine:2.9.3") {
97+
because("Last version to support JDK 8.")
98+
}
5299

53100
testImplementation("ch.qos.logback:logback-classic:1.5.12")
54101
testImplementation("org.junit.jupiter:junit-jupiter:5.11.3")
@@ -140,6 +187,10 @@ tasks {
140187
}
141188
}
142189

190+
named<Task>("check") {
191+
dependsOn(named<Task>("testIntegration"))
192+
}
193+
143194
named<Task>("jacocoTestReport") {
144195
group = "jacoco"
145196
dependsOn(named("test")) // tests are required to run before generating the report
@@ -162,32 +213,10 @@ tasks {
162213
group = "sonar"
163214
}
164215

165-
named<Test>("test") {
166-
description = "Run unit tests"
167-
outputs.upToDateWhen { false }
168-
useJUnitPlatform {
169-
filter {
170-
excludeTestsMatching("*IT")
171-
}
172-
}
173-
}
174-
175216
withType<Test> {
176217
defaultCharacterEncoding = "UTF-8"
177218
}
178219

179-
create<Test>("it-s3") {
180-
group = "verification"
181-
description = "Run integration tests using S3"
182-
useJUnitPlatform {
183-
filter {
184-
includeTestsMatching("*IT")
185-
includeTags("it-s3")
186-
}
187-
}
188-
mustRunAfter(named("test"))
189-
}
190-
191220
// TODO: There are some problems with using minio that overcomplicate the setup.
192221
// For the time being we'll be disabling it until we figure out the best path forward.
193222
// create<Test>("it-minio") {
@@ -201,10 +230,6 @@ tasks {
201230
// }
202231
// }
203232

204-
named<Task>("check") {
205-
dependsOn(named("it-s3"))
206-
}
207-
208233
withType<Sign> {
209234
onlyIf {
210235
(project.hasProperty("withSignature") && project.findProperty("withSignature") == "true") ||

‎docs/content/contributing/developer-guide/index.md

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ Before you start writing code, please read:
1010
## System requirements
1111

1212
1. Gradle 8.1, or higher
13-
2. `JDK8`, `JDK11` or `JDK17`
13+
2. `JDK8`, `JDK11`, `JDK17` or `JDK21`
1414

1515
## Finding issues to work on
1616

@@ -85,7 +85,7 @@ s3fs.proxy.url=https://my.local.domain/path/to/repository
8585
### Build
8686
8787
Builds the entire code and runs unit and integration tests.
88-
It is assumed you already have the `amazon-test.properties` configuration in place.
88+
It is assumed you already have the `amazon-test.properties` configuration in place under the `src/test/resources` or `src/testIntegration/resources`.
8989
9090
```
9191
./gradlew build
@@ -100,9 +100,11 @@ It is assumed you already have the `amazon-test.properties` configuration in pla
100100
### Run only integration tests
101101
102102
```
103-
./gradlew it-s3
103+
./gradlew testIntegration
104104
```
105105
106+
You can also use `./gradlew build -x testIntegration` to skip the integration tests.
107+
106108
### Run all tests
107109
108110
```

‎docs/content/reference/configuration-options.md

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,28 +4,30 @@
44

55
A complete list of environment variables which can be set to configure the client.
66

7-
| Key | Default | Description |
8-
|-------------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------|
9-
| s3fs.access.key | none | <small>AWS access key, used to identify the user interacting with AWS</small> |
10-
| s3fs.secret.key | none | <small>AWS secret access key, used to authenticate the user interacting with AWS</small> |
11-
| s3fs.request.metric.collector.class | TODO | <small>Fully-qualified class name to instantiate an AWS SDK request/response metric collector</small> |
12-
| s3fs.connection.timeout | TODO | <small>Timeout (in milliseconds) for establishing a connection to a remote service</small> |
13-
| s3fs.max.connections | TODO | <small>Maximum number of connections allowed in a connection pool</small> |
14-
| s3fs.max.retry.error | TODO | <small>Maximum number of times that a single request should be retried, assuming it fails for a retryable error</small> |
15-
| s3fs.protocol | TODO | <small>Protocol (HTTP or HTTPS) to use when connecting to AWS</small> |
16-
| s3fs.proxy.domain | none | <small>For NTLM proxies: The Windows domain name to use when authenticating with the proxy</small> |
17-
| s3fs.proxy.protocol | none | <small>Proxy connection protocol.</small> |
18-
| s3fs.proxy.host | none | <small>Proxy host name either from the configured endpoint or from the "http.proxyHost" system property</small> |
19-
| s3fs.proxy.password | none | <small>The password to use when connecting through a proxy</small> |
20-
| s3fs.proxy.port | none | <small>Proxy port either from the configured endpoint or from the "http.proxyPort" system property</small> |
21-
| s3fs.proxy.username | none | <small>The username to use when connecting through a proxy</small> |
22-
| s3fs.proxy.workstation | none | <small>For NTLM proxies: The Windows workstation name to use when authenticating with the proxy</small> |
23-
| s3fs.region | none | <small>The AWS Region to configure the client</small> |
24-
| s3fs.socket.send.buffer.size.hint | TODO | <small>The size hint (in bytes) for the low level TCP send buffer</small> |
25-
| s3fs.socket.receive.buffer.size.hint | TODO | <small>The size hint (in bytes) for the low level TCP receive buffer</small> |
26-
| s3fs.socket.timeout | TODO | <small>Timeout (in milliseconds) for each read to the underlying socket</small> |
27-
| s3fs.user.agent.prefix | TODO | <small>Prefix of the user agent that is sent with each request to AWS</small> |
28-
| s3fs.amazon.s3.factory.class | TODO | <small>Fully-qualified class name to instantiate a S3 factory base class which creates a S3 client instance</small> |
29-
| s3fs.signer.override | TODO | <small>Fully-qualified class name to define the signer that should be used when authenticating with AWS</small> |
30-
| s3fs.path.style.access | TODO | <small>Boolean that indicates whether the client uses path-style access for all requests</small> |
31-
| s3fs.request.header.cache-control | blank | <small>Configures the `cacheControl` on request builders (i.e. `CopyObjectRequest`, `PutObjectRequest`, etc) |
7+
| Key | Default | Description |
8+
|-------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------|
9+
| s3fs.access.key | none | <small>AWS access key, used to identify the user interacting with AWS</small> |
10+
| s3fs.secret.key | none | <small>AWS secret access key, used to authenticate the user interacting with AWS</small> |
11+
| s3fs.request.metric.collector.class | TODO | <small>Fully-qualified class name to instantiate an AWS SDK request/response metric collector</small> |
12+
| s3fs.cache.attributes.ttl | `60000` | <small>TTL for the cached file attributes (in millis)</small> |
13+
| s3fs.cache.attributes.size | `5000` | <small>Total size of cached file attributes</small> |
14+
| s3fs.connection.timeout | TODO | <small>Timeout (in milliseconds) for establishing a connection to a remote service</small> |
15+
| s3fs.max.connections | TODO | <small>Maximum number of connections allowed in a connection pool</small> |
16+
| s3fs.max.retry.error | TODO | <small>Maximum number of times that a single request should be retried, assuming it fails for a retryable error</small> |
17+
| s3fs.protocol | TODO | <small>Protocol (HTTP or HTTPS) to use when connecting to AWS</small> |
18+
| s3fs.proxy.domain | none | <small>For NTLM proxies: The Windows domain name to use when authenticating with the proxy</small> |
19+
| s3fs.proxy.protocol | none | <small>Proxy connection protocol.</small> |
20+
| s3fs.proxy.host | none | <small>Proxy host name either from the configured endpoint or from the "http.proxyHost" system property</small> |
21+
| s3fs.proxy.password | none | <small>The password to use when connecting through a proxy</small> |
22+
| s3fs.proxy.port | none | <small>Proxy port either from the configured endpoint or from the "http.proxyPort" system property</small> |
23+
| s3fs.proxy.username | none | <small>The username to use when connecting through a proxy</small> |
24+
| s3fs.proxy.workstation | none | <small>For NTLM proxies: The Windows workstation name to use when authenticating with the proxy</small> |
25+
| s3fs.region | none | <small>The AWS Region to configure the client</small> |
26+
| s3fs.socket.send.buffer.size.hint | TODO | <small>The size hint (in bytes) for the low level TCP send buffer</small> |
27+
| s3fs.socket.receive.buffer.size.hint | TODO | <small>The size hint (in bytes) for the low level TCP receive buffer</small> |
28+
| s3fs.socket.timeout | TODO | <small>Timeout (in milliseconds) for each read to the underlying socket</small> |
29+
| s3fs.user.agent.prefix | TODO | <small>Prefix of the user agent that is sent with each request to AWS</small> |
30+
| s3fs.amazon.s3.factory.class | TODO | <small>Fully-qualified class name to instantiate a S3 factory base class which creates a S3 client instance</small> |
31+
| s3fs.signer.override | TODO | <small>Fully-qualified class name to define the signer that should be used when authenticating with AWS</small> |
32+
| s3fs.path.style.access | TODO | <small>Boolean that indicates whether the client uses path-style access for all requests</small> |
33+
| s3fs.request.header.cache-control | blank | <small>Configures the `cacheControl` on request builders (i.e. `CopyObjectRequest`, `PutObjectRequest`, etc) |

‎src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import java.time.Duration;
66
import java.util.Properties;
77

8+
import org.carlspring.cloud.storage.s3fs.attribute.S3BasicFileAttributes;
9+
import org.carlspring.cloud.storage.s3fs.attribute.S3PosixFileAttributes;
810
import org.slf4j.Logger;
911
import org.slf4j.LoggerFactory;
1012
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
@@ -41,6 +43,18 @@ public abstract class S3Factory
4143

4244
public static final String SECRET_KEY = "s3fs.secret.key";
4345

46+
/**
47+
* Maximum TTL in millis to cache {@link S3BasicFileAttributes} and {@link S3PosixFileAttributes}.
48+
*/
49+
public static final String CACHE_ATTRIBUTES_TTL = "s3fs.cache.attributes.ttl";
50+
public static final int CACHE_ATTRIBUTES_TTL_DEFAULT = 60000;
51+
52+
/**
53+
* Total size of {@link S3BasicFileAttributes} and {@link S3PosixFileAttributes} cache.
54+
*/
55+
public static final String CACHE_ATTRIBUTES_SIZE = "s3fs.cache.attributes.size";
56+
public static final int CACHE_ATTRIBUTES_SIZE_DEFAULT = 30000;
57+
4458
public static final String REQUEST_METRIC_COLLECTOR_CLASS = "s3fs.request.metric.collector.class";
4559

4660
public static final String CONNECTION_TIMEOUT = "s3fs.connection.timeout";

‎src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystem.java

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
package org.carlspring.cloud.storage.s3fs;
22

3+
import com.google.common.collect.ImmutableList;
4+
import com.google.common.collect.ImmutableSet;
5+
import org.carlspring.cloud.storage.s3fs.cache.S3FileAttributesCache;
6+
import org.carlspring.cloud.storage.s3fs.util.S3Utils;
7+
import software.amazon.awssdk.services.s3.S3Client;
8+
import software.amazon.awssdk.services.s3.model.Bucket;
9+
310
import java.io.IOException;
411
import java.nio.file.FileStore;
512
import java.nio.file.FileSystem;
@@ -10,10 +17,6 @@
1017
import java.util.Properties;
1118
import java.util.Set;
1219

13-
import com.google.common.collect.ImmutableList;
14-
import com.google.common.collect.ImmutableSet;
15-
import software.amazon.awssdk.services.s3.S3Client;
16-
import software.amazon.awssdk.services.s3.model.Bucket;
1720
import static org.carlspring.cloud.storage.s3fs.S3Path.PATH_SEPARATOR;
1821

1922
/**
@@ -34,7 +37,7 @@ public class S3FileSystem
3437

3538
private final String endpoint;
3639

37-
private final int cache;
40+
private S3FileAttributesCache fileAttributesCache;
3841

3942
private final Properties properties;
4043

@@ -48,8 +51,12 @@ public S3FileSystem(final S3FileSystemProvider provider,
4851
this.key = key;
4952
this.client = client;
5053
this.endpoint = endpoint;
51-
this.cache = 60000; // 1 minute cache for the s3Path
5254
this.properties = properties;
55+
56+
int cacheTTL = Integer.parseInt(String.valueOf(properties.getOrDefault(S3Factory.CACHE_ATTRIBUTES_TTL, S3Factory.CACHE_ATTRIBUTES_TTL_DEFAULT)));
57+
int cacheSize = Integer.parseInt(String.valueOf(properties.getOrDefault(S3Factory.CACHE_ATTRIBUTES_SIZE, S3Factory.CACHE_ATTRIBUTES_SIZE_DEFAULT)));
58+
59+
this.fileAttributesCache = new S3FileAttributesCache(cacheTTL, cacheSize);
5360
}
5461

5562
public S3FileSystem(final S3FileSystemProvider provider,
@@ -75,6 +82,7 @@ public String getKey()
7582
public void close()
7683
throws IOException
7784
{
85+
this.fileAttributesCache.invalidateAll();
7886
this.provider.close(this);
7987
}
8088

@@ -171,14 +179,22 @@ public String getEndpoint()
171179
return endpoint;
172180
}
173181

182+
/**
183+
* @deprecated Use {@link org.carlspring.cloud.storage.s3fs.util.S3Utils#key2Parts(String)} instead. To be removed in one of next majors versions.
184+
* @param keyParts
185+
* @return String[]
186+
*/
174187
public String[] key2Parts(String keyParts)
175188
{
176-
return keyParts.split(PATH_SEPARATOR);
189+
return S3Utils.key2Parts(keyParts);
177190
}
178191

179-
public int getCache()
192+
/**
193+
* @return The {@link S3FileAttributesCache} instance holding the path attributes cache for this file provider.
194+
*/
195+
public S3FileAttributesCache getFileAttributesCache()
180196
{
181-
return cache;
197+
return fileAttributesCache;
182198
}
183199

184200
/**

0 commit comments

Comments
 (0)