Skip to content

Commit 0f06e0e

Browse files
nknizenknize
committed
[DEPRECATE] SimpleFS in favor of NIOFS (opensearch-project#1073)
Lucene 9 removes support for SimpleFS File System format. This commit deprecates the SimpleFS format in favor of NIOFS. Signed-off-by: Nicholas Walter Knize <[email protected]>
1 parent 55c1026 commit 0f06e0e

File tree

24 files changed

+209
-51
lines changed

24 files changed

+209
-51
lines changed

‎distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreWrapperTests.java

+7-7
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
import org.apache.lucene.codecs.CodecUtil;
3636
import org.apache.lucene.store.IOContext;
3737
import org.apache.lucene.store.IndexOutput;
38-
import org.apache.lucene.store.SimpleFSDirectory;
38+
import org.apache.lucene.store.NIOFSDirectory;
3939
import org.opensearch.common.Randomness;
4040
import org.opensearch.core.internal.io.IOUtils;
4141
import org.opensearch.env.Environment;
@@ -192,7 +192,7 @@ public void testUpgradeNoop() throws Exception {
192192

193193
public void testFailWhenCannotConsumeSecretStream() throws Exception {
194194
Path configDir = env.configFile();
195-
SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
195+
NIOFSDirectory directory = new NIOFSDirectory(configDir);
196196
try (IndexOutput indexOutput = directory.createOutput("opensearch.keystore", IOContext.DEFAULT)) {
197197
CodecUtil.writeHeader(indexOutput, "opensearch.keystore", 3);
198198
indexOutput.writeByte((byte) 0); // No password
@@ -220,7 +220,7 @@ public void testFailWhenCannotConsumeSecretStream() throws Exception {
220220

221221
public void testFailWhenCannotConsumeEncryptedBytesStream() throws Exception {
222222
Path configDir = env.configFile();
223-
SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
223+
NIOFSDirectory directory = new NIOFSDirectory(configDir);
224224
try (IndexOutput indexOutput = directory.createOutput("opensearch.keystore", IOContext.DEFAULT)) {
225225
CodecUtil.writeHeader(indexOutput, "opensearch.keystore", 3);
226226
indexOutput.writeByte((byte) 0); // No password
@@ -249,7 +249,7 @@ public void testFailWhenCannotConsumeEncryptedBytesStream() throws Exception {
249249

250250
public void testFailWhenSecretStreamNotConsumed() throws Exception {
251251
Path configDir = env.configFile();
252-
SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
252+
NIOFSDirectory directory = new NIOFSDirectory(configDir);
253253
try (IndexOutput indexOutput = directory.createOutput("opensearch.keystore", IOContext.DEFAULT)) {
254254
CodecUtil.writeHeader(indexOutput, "opensearch.keystore", 3);
255255
indexOutput.writeByte((byte) 0); // No password
@@ -276,7 +276,7 @@ public void testFailWhenSecretStreamNotConsumed() throws Exception {
276276

277277
public void testFailWhenEncryptedBytesStreamIsNotConsumed() throws Exception {
278278
Path configDir = env.configFile();
279-
SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
279+
NIOFSDirectory directory = new NIOFSDirectory(configDir);
280280
try (IndexOutput indexOutput = directory.createOutput("opensearch.keystore", IOContext.DEFAULT)) {
281281
CodecUtil.writeHeader(indexOutput, "opensearch.keystore", 3);
282282
indexOutput.writeByte((byte) 0); // No password
@@ -362,7 +362,7 @@ public void testIllegalSettingName() throws Exception {
362362
public void testBackcompatV1() throws Exception {
363363
assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
364364
Path configDir = env.configFile();
365-
SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
365+
NIOFSDirectory directory = new NIOFSDirectory(configDir);
366366
try (IndexOutput output = directory.createOutput("opensearch.keystore", IOContext.DEFAULT)) {
367367
CodecUtil.writeHeader(output, "opensearch.keystore", 1);
368368
output.writeByte((byte) 0); // hasPassword = false
@@ -393,7 +393,7 @@ public void testBackcompatV1() throws Exception {
393393
public void testBackcompatV2() throws Exception {
394394
assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
395395
Path configDir = env.configFile();
396-
SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
396+
NIOFSDirectory directory = new NIOFSDirectory(configDir);
397397
byte[] fileBytes = new byte[20];
398398
random().nextBytes(fileBytes);
399399
try (IndexOutput output = directory.createOutput("opensearch.keystore", IOContext.DEFAULT)) {

‎plugins/store-smb/src/internalClusterTest/java/org/opensearch/index/store/SmbNIOFsTests.java

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.index.store;
10+
11+
import org.opensearch.common.settings.Settings;
12+
13+
/**
14+
* Index Settings Tests for NIO FileSystem as index store type.
15+
*/
16+
public class SmbNIOFsTests extends AbstractAzureFsTestCase {
17+
@Override
18+
public Settings indexSettings() {
19+
return Settings.builder()
20+
.put(super.indexSettings())
21+
.put("index.store.type", "smb_nio_fs")
22+
.build();
23+
}
24+
}

‎plugins/store-smb/src/main/java/org/opensearch/index/store/smbniofs/SmbNIOFsDirectoryFactory.java

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.index.store.smbniofs;
10+
11+
import org.apache.lucene.store.Directory;
12+
import org.apache.lucene.store.LockFactory;
13+
import org.apache.lucene.store.NIOFSDirectory;
14+
import org.opensearch.index.IndexSettings;
15+
import org.opensearch.index.store.FsDirectoryFactory;
16+
import org.opensearch.index.store.SmbDirectoryWrapper;
17+
18+
import java.io.IOException;
19+
import java.nio.file.Path;
20+
21+
/**
22+
* Factory to create a new NIO File System type directory accessible as a SMB share
23+
*/
24+
public final class SmbNIOFsDirectoryFactory extends FsDirectoryFactory {
25+
26+
@Override
27+
protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException {
28+
return new SmbDirectoryWrapper(new NIOFSDirectory(location, lockFactory));
29+
}
30+
}

‎plugins/store-smb/src/main/java/org/opensearch/index/store/smbsimplefs/SmbSimpleFsDirectoryFactory.java

+12
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,29 @@
3535
import org.apache.lucene.store.Directory;
3636
import org.apache.lucene.store.LockFactory;
3737
import org.apache.lucene.store.SimpleFSDirectory;
38+
import org.opensearch.common.logging.DeprecationLogger;
39+
import org.opensearch.index.IndexModule;
3840
import org.opensearch.index.IndexSettings;
3941
import org.opensearch.index.store.FsDirectoryFactory;
4042
import org.opensearch.index.store.SmbDirectoryWrapper;
4143

4244
import java.io.IOException;
4345
import java.nio.file.Path;
4446

47+
/**
48+
* Factory to create a new Simple File System type directory accessible as a SMB share
49+
*
50+
* @deprecated use {@link org.opensearch.index.store.smbniofs.SmbNIOFsDirectoryFactory} instead
51+
*/
52+
@Deprecated
4553
public final class SmbSimpleFsDirectoryFactory extends FsDirectoryFactory {
4654

55+
private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(SmbSimpleFsDirectoryFactory.class);
56+
4757
@Override
4858
protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException {
59+
DEPRECATION_LOGGER.deprecate(IndexModule.Type.SIMPLEFS.getSettingsKey(), IndexModule.Type.SIMPLEFS.getSettingsKey()
60+
+ " is deprecated and will be removed in 2.0");
4961
return new SmbDirectoryWrapper(new SimpleFSDirectory(location, lockFactory));
5062
}
5163
}

‎plugins/store-smb/src/main/java/org/opensearch/plugin/store/smb/SMBStorePlugin.java

+8
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,10 @@
3232

3333
package org.opensearch.plugin.store.smb;
3434

35+
import org.opensearch.common.logging.DeprecationLogger;
36+
import org.opensearch.index.IndexModule;
3537
import org.opensearch.index.store.smbmmapfs.SmbMmapFsDirectoryFactory;
38+
import org.opensearch.index.store.smbniofs.SmbNIOFsDirectoryFactory;
3639
import org.opensearch.index.store.smbsimplefs.SmbSimpleFsDirectoryFactory;
3740
import org.opensearch.plugins.IndexStorePlugin;
3841
import org.opensearch.plugins.Plugin;
@@ -43,11 +46,16 @@
4346

4447
public class SMBStorePlugin extends Plugin implements IndexStorePlugin {
4548

49+
private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(SMBStorePlugin.class);
50+
4651
@Override
4752
public Map<String, DirectoryFactory> getDirectoryFactories() {
4853
final Map<String, DirectoryFactory> indexStoreFactories = new HashMap<>(2);
4954
indexStoreFactories.put("smb_mmap_fs", new SmbMmapFsDirectoryFactory());
55+
DEPRECATION_LOGGER.deprecate(IndexModule.Type.SIMPLEFS.getSettingsKey(), IndexModule.Type.SIMPLEFS.getSettingsKey()
56+
+ " is deprecated and will be removed in 2.0");
5057
indexStoreFactories.put("smb_simple_fs", new SmbSimpleFsDirectoryFactory());
58+
indexStoreFactories.put("smb_nio_fs", new SmbNIOFsDirectoryFactory());
5159
return Collections.unmodifiableMap(indexStoreFactories);
5260
}
5361

‎plugins/store-smb/src/test/java/org/opensearch/index/store/SmbNIOFSDirectoryTests.java

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.index.store;
10+
11+
import org.apache.lucene.store.Directory;
12+
import org.apache.lucene.store.NIOFSDirectory;
13+
14+
import java.io.IOException;
15+
import java.nio.file.Path;
16+
17+
/**
18+
* SMB Tests using NIO FileSystem as index store type.
19+
*/
20+
public class SmbNIOFSDirectoryTests extends OpenSearchBaseDirectoryTestCase {
21+
22+
@Override
23+
protected Directory getDirectory(Path file) throws IOException {
24+
return new SmbDirectoryWrapper(new NIOFSDirectory(file));
25+
}
26+
27+
@Override
28+
public void testCreateOutputForExistingFile() throws IOException {
29+
/**
30+
* This test is disabled because {@link SmbDirectoryWrapper} opens existing file
31+
* with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
32+
*/
33+
}
34+
}

‎server/src/internalClusterTest/java/org/opensearch/cluster/routing/AllocationIdIT.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232

3333
package org.opensearch.cluster.routing;
3434

35-
import org.apache.lucene.store.SimpleFSDirectory;
35+
import org.apache.lucene.store.NIOFSDirectory;
3636

3737
import org.opensearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
3838
import org.opensearch.action.admin.indices.stats.ShardStats;
@@ -151,7 +151,7 @@ public void testFailedRecoveryOnAllocateStalePrimaryRequiresAnotherAllocateStale
151151
});
152152

153153
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1));
154-
try(Store store = new Store(shardId, indexSettings, new SimpleFSDirectory(indexPath), new DummyShardLock(shardId))) {
154+
try(Store store = new Store(shardId, indexSettings, new NIOFSDirectory(indexPath), new DummyShardLock(shardId))) {
155155
store.removeCorruptionMarker();
156156
}
157157
node1 = internalCluster().startNode(node1DataPathSettings);
@@ -229,7 +229,7 @@ private String historyUUID(String node, String indexName) {
229229
}
230230

231231
private void putFakeCorruptionMarker(IndexSettings indexSettings, ShardId shardId, Path indexPath) throws IOException {
232-
try(Store store = new Store(shardId, indexSettings, new SimpleFSDirectory(indexPath), new DummyShardLock(shardId))) {
232+
try(Store store = new Store(shardId, indexSettings, new NIOFSDirectory(indexPath), new DummyShardLock(shardId))) {
233233
store.markStoreCorrupted(new IOException("fake ioexception"));
234234
}
235235
}

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

+12
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
import org.opensearch.common.xcontent.XContentHelper;
7777
import org.opensearch.env.Environment;
7878
import org.opensearch.index.Index;
79+
import org.opensearch.index.IndexModule;
7980
import org.opensearch.index.IndexNotFoundException;
8081
import org.opensearch.index.IndexService;
8182
import org.opensearch.index.IndexSettings;
@@ -783,9 +784,20 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
783784
"Please do not specify value for setting [index.soft_deletes.enabled] of index [" + request.index() + "].");
784785
}
785786
validateTranslogRetentionSettings(indexSettings);
787+
validateStoreTypeSettings(indexSettings);
788+
786789
return indexSettings;
787790
}
788791

792+
public static void validateStoreTypeSettings(Settings settings) {
793+
// deprecate simplefs store type:
794+
if (IndexModule.Type.SIMPLEFS.match(IndexModule.INDEX_STORE_TYPE_SETTING.get(settings))) {
795+
DEPRECATION_LOGGER.deprecate("store_type_setting",
796+
"[simplefs] is deprecated and will be removed in 2.0. Use [niofs], which offers equal or better performance, " +
797+
"or other file systems instead.");
798+
}
799+
}
800+
789801
static int getNumberOfShards(final Settings.Builder indexSettingsBuilder) {
790802
// TODO: this logic can be removed when the current major version is 8
791803
assert Version.CURRENT.major == 1;

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -250,8 +250,12 @@ public ClusterState execute(ClusterState currentState) {
250250

251251
if (IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.exists(normalizedSettings) ||
252252
IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(normalizedSettings)) {
253+
Settings indexSettings;
253254
for (String index : actualIndices) {
254-
MetadataCreateIndexService.validateTranslogRetentionSettings(metadataBuilder.get(index).getSettings());
255+
indexSettings = metadataBuilder.get(index).getSettings();
256+
MetadataCreateIndexService.validateTranslogRetentionSettings(indexSettings);
257+
// validate storeType for deprecating index stores
258+
MetadataCreateIndexService.validateStoreTypeSettings(indexSettings);
255259
}
256260
}
257261
boolean changed = false;

‎server/src/main/java/org/opensearch/common/settings/KeyStoreWrapper.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
import org.apache.lucene.store.IOContext;
4141
import org.apache.lucene.store.IndexInput;
4242
import org.apache.lucene.store.IndexOutput;
43-
import org.apache.lucene.store.SimpleFSDirectory;
43+
import org.apache.lucene.store.NIOFSDirectory;
4444
import org.apache.lucene.util.SetOnce;
4545
import org.opensearch.cli.ExitCodes;
4646
import org.opensearch.cli.UserException;
@@ -229,7 +229,7 @@ public static KeyStoreWrapper load(Path configDir) throws IOException {
229229
return null;
230230
}
231231

232-
SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
232+
NIOFSDirectory directory = new NIOFSDirectory(configDir);
233233
try (IndexInput indexInput = directory.openInput(KEYSTORE_FILENAME, IOContext.READONCE)) {
234234
ChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput);
235235
final int formatVersion;
@@ -495,7 +495,7 @@ private void decryptLegacyEntries() throws GeneralSecurityException, IOException
495495
public synchronized void save(Path configDir, char[] password) throws Exception {
496496
ensureOpen();
497497

498-
SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
498+
NIOFSDirectory directory = new NIOFSDirectory(configDir);
499499
// write to tmp file first, then overwrite
500500
String tmpFile = KEYSTORE_FILENAME + ".tmp";
501501
try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) {

‎server/src/main/java/org/opensearch/env/NodeEnvironment.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@
4242
import org.apache.lucene.store.FSDirectory;
4343
import org.apache.lucene.store.Lock;
4444
import org.apache.lucene.store.LockObtainFailedException;
45+
import org.apache.lucene.store.NIOFSDirectory;
4546
import org.apache.lucene.store.NativeFSLockFactory;
46-
import org.apache.lucene.store.SimpleFSDirectory;
4747
import org.opensearch.OpenSearchException;
4848
import org.opensearch.Version;
4949
import org.opensearch.cluster.metadata.IndexMetadata;
@@ -501,7 +501,7 @@ public static void acquireFSLockForPaths(IndexSettings indexSettings, Path... sh
501501
// resolve the directory the shard actually lives in
502502
Path p = shardPaths[i].resolve("index");
503503
// open a directory (will be immediately closed) on the shard's location
504-
dirs[i] = new SimpleFSDirectory(p, indexSettings.getValue(FsDirectoryFactory.INDEX_LOCK_FACTOR_SETTING));
504+
dirs[i] = new NIOFSDirectory(p, indexSettings.getValue(FsDirectoryFactory.INDEX_LOCK_FACTOR_SETTING));
505505
// create a lock for the "write.lock" file
506506
try {
507507
locks[i] = dirs[i].obtainLock(IndexWriter.WRITE_LOCK_NAME);

‎server/src/main/java/org/opensearch/gateway/MetadataStateFormat.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
import org.apache.lucene.store.IOContext;
4343
import org.apache.lucene.store.IndexInput;
4444
import org.apache.lucene.store.IndexOutput;
45-
import org.apache.lucene.store.SimpleFSDirectory;
45+
import org.apache.lucene.store.NIOFSDirectory;
4646
import org.opensearch.ExceptionsHelper;
4747
import org.opensearch.common.collect.Tuple;
4848
import org.opensearch.common.lucene.store.IndexOutputOutputStream;
@@ -323,7 +323,7 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro
323323
}
324324

325325
protected Directory newDirectory(Path dir) throws IOException {
326-
return new SimpleFSDirectory(dir);
326+
return new NIOFSDirectory(dir);
327327
}
328328

329329

‎server/src/main/java/org/opensearch/gateway/PersistedClusterStateService.java

+7-7
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@
5656
import org.apache.lucene.search.Weight;
5757
import org.apache.lucene.store.AlreadyClosedException;
5858
import org.apache.lucene.store.Directory;
59-
import org.apache.lucene.store.SimpleFSDirectory;
59+
import org.apache.lucene.store.NIOFSDirectory;
6060
import org.apache.lucene.util.Bits;
6161
import org.apache.lucene.util.BytesRef;
6262
import org.apache.lucene.util.BytesRefIterator;
@@ -228,16 +228,16 @@ private static IndexWriter createIndexWriter(Directory directory, boolean openEx
228228
*/
229229
public static void deleteAll(Path[] dataPaths) throws IOException {
230230
for (Path dataPath : dataPaths) {
231-
Lucene.cleanLuceneIndex(new SimpleFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)));
231+
Lucene.cleanLuceneIndex(new NIOFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)));
232232
}
233233
}
234234

235235
// exposed for tests
236236
Directory createDirectory(Path path) throws IOException {
237237
// it is possible to disable the use of MMapDirectory for indices, and it may be surprising to users that have done so if we still
238238
// use a MMapDirectory here, which might happen with FSDirectory.open(path). Concurrency is of no concern here so a
239-
// SimpleFSDirectory is fine:
240-
return new SimpleFSDirectory(path);
239+
// NIOFSDirectory is fine:
240+
return new NIOFSDirectory(path);
241241
}
242242

243243
public Path[] getDataPaths() {
@@ -277,7 +277,7 @@ public static NodeMetadata nodeMetadata(Path... dataPaths) throws IOException {
277277
for (final Path dataPath : dataPaths) {
278278
final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME);
279279
if (Files.exists(indexPath)) {
280-
try (DirectoryReader reader = DirectoryReader.open(new SimpleFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)))) {
280+
try (DirectoryReader reader = DirectoryReader.open(new NIOFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)))) {
281281
final Map<String, String> userData = reader.getIndexCommit().getUserData();
282282
assert userData.get(NODE_VERSION_KEY) != null;
283283

@@ -308,12 +308,12 @@ public static void overrideVersion(Version newVersion, Path... dataPaths) throws
308308
for (final Path dataPath : dataPaths) {
309309
final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME);
310310
if (Files.exists(indexPath)) {
311-
try (DirectoryReader reader = DirectoryReader.open(new SimpleFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)))) {
311+
try (DirectoryReader reader = DirectoryReader.open(new NIOFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)))) {
312312
final Map<String, String> userData = reader.getIndexCommit().getUserData();
313313
assert userData.get(NODE_VERSION_KEY) != null;
314314

315315
try (IndexWriter indexWriter =
316-
createIndexWriter(new SimpleFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)), true)) {
316+
createIndexWriter(new NIOFSDirectory(dataPath.resolve(METADATA_DIRECTORY_NAME)), true)) {
317317
final Map<String, String> commitData = new HashMap<>(userData);
318318
commitData.put(NODE_VERSION_KEY, Integer.toString(newVersion.id));
319319
indexWriter.setLiveCommitData(commitData.entrySet());

0 commit comments

Comments
 (0)