Skip to content

Commit d0921bc

Browse files
wildeslamwildeslam
wildeslam
and
wildeslam
authored
feat: unified mgnt of mongo indexes (#165)
* feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes * feat: unified mgnt of mongo indexes --------- Co-authored-by: wildeslam <[email protected]>
1 parent 0619d37 commit d0921bc

File tree

5 files changed

+435
-168
lines changed

5 files changed

+435
-168
lines changed

‎arex-storage-web-api/src/main/java/com/arextest/storage/beans/IndexesSettingConfiguration.java

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
package com.arextest.storage.beans;
2+
3+
import com.arextest.common.cache.CacheProvider;
4+
import com.arextest.common.cache.LockWrapper;
5+
import com.arextest.model.mock.AREXMocker;
6+
import com.arextest.model.mock.MockCategoryType;
7+
import com.arextest.storage.cache.CacheKeyUtils;
8+
import com.arextest.storage.enums.MongoCollectionIndexConfigEnum;
9+
import com.arextest.storage.enums.MongoCollectionIndexConfigEnum.FieldConfig;
10+
import com.arextest.storage.enums.MongoCollectionIndexConfigEnum.TtlIndexConfig;
11+
import com.arextest.storage.repository.ProviderNames;
12+
import com.mongodb.MongoCommandException;
13+
import com.mongodb.client.ListIndexesIterable;
14+
import com.mongodb.client.MongoCollection;
15+
import com.mongodb.client.MongoDatabase;
16+
import com.mongodb.client.model.IndexOptions;
17+
import java.lang.reflect.Field;
18+
import java.util.ArrayList;
19+
import java.util.List;
20+
import java.util.Objects;
21+
import java.util.concurrent.TimeUnit;
22+
import javax.annotation.Resource;
23+
import lombok.extern.slf4j.Slf4j;
24+
import org.apache.commons.lang3.tuple.Pair;
25+
import org.bson.Document;
26+
import org.bson.conversions.Bson;
27+
import org.springframework.context.annotation.Configuration;
28+
29+
@Slf4j
30+
@Configuration(proxyBeanMethods = false)
31+
public class IndexesSettingConfiguration {
32+
33+
private static final String EXPIRATION_TIME_COLUMN_NAME = "expirationTime";
34+
private static final String COLLECTION_SUFFIX = "Mocker";
35+
private static final String BACKGROUND = "background";
36+
private static final String UNIQUE = "unique";
37+
private static final String EXPIRE_AFTER_SECONDS = "expireAfterSeconds";
38+
private static final String ID = "_id_";
39+
private static final String KEY = "key";
40+
private static final String NAME = "name";
41+
private static final String REDIS_KEY_VERSION = "storage_version";
42+
43+
// increment this version when you want to recreate indexes
44+
private static final int INDEX_VERSION = 1;
45+
46+
@Resource
47+
private CacheProvider cacheProvider;
48+
49+
public void setIndexes(MongoDatabase mongoDatabase) {
50+
Runnable runnable = () -> {
51+
LockWrapper lock = cacheProvider.getLock("setIndexes");
52+
try {
53+
try {
54+
lock.lock(3, TimeUnit.SECONDS);
55+
byte[] versionKey = CacheKeyUtils.toUtf8Bytes(REDIS_KEY_VERSION);
56+
String versionValue = CacheKeyUtils.fromUtf8Bytes(cacheProvider.get(versionKey));
57+
if (versionValue != null && INDEX_VERSION == Integer.parseInt(versionValue)) {
58+
LOGGER.info("indexes already set");
59+
return;
60+
} else {
61+
cacheProvider.put(versionKey, CacheKeyUtils.toUtf8Bytes(String.valueOf(INDEX_VERSION)));
62+
}
63+
} catch (Exception e) {
64+
LOGGER.error("set version failed", e);
65+
return;
66+
} finally {
67+
lock.unlock();
68+
}
69+
70+
71+
long timestamp = System.currentTimeMillis();
72+
LOGGER.info("start to set indexes");
73+
for (MongoCollectionIndexConfigEnum indexConfigEnum : MongoCollectionIndexConfigEnum.values()) {
74+
try {
75+
setIndexByEnum(indexConfigEnum, mongoDatabase);
76+
} catch (Exception e) {
77+
LOGGER.error("set index failed for {}", indexConfigEnum.getCollectionName(), e);
78+
}
79+
}
80+
ensureMockerQueryIndex(mongoDatabase);
81+
LOGGER.info("set indexes success. cost: {}ms", System.currentTimeMillis() - timestamp);
82+
} catch (Exception e) {
83+
LOGGER.error("set indexes failed", e);
84+
}
85+
};
86+
Thread thread = new Thread(runnable);
87+
thread.start();
88+
}
89+
90+
private void ensureMockerQueryIndex(MongoDatabase database) {
91+
for (MockCategoryType category : MockCategoryType.DEFAULTS) {
92+
for (Field field : ProviderNames.class.getDeclaredFields()) {
93+
String providerName = null;
94+
try {
95+
providerName = (String) field.get(ProviderNames.class);
96+
} catch (IllegalAccessException e) {
97+
LOGGER.error("get provider name failed", e);
98+
continue;
99+
}
100+
101+
MongoCollection<AREXMocker> collection =
102+
database.getCollection(getCollectionName(category, providerName),
103+
AREXMocker.class);
104+
try {
105+
Document index = new Document();
106+
index.append(AREXMocker.Fields.recordId, 1);
107+
collection.createIndex(index);
108+
} catch (MongoCommandException e) {
109+
LOGGER.info("create index failed for {}", category.getName(), e);
110+
}
111+
112+
try {
113+
Document index = new Document();
114+
index.append(AREXMocker.Fields.appId, 1);
115+
index.append(AREXMocker.Fields.operationName, 1);
116+
collection.createIndex(index);
117+
} catch (MongoCommandException e) {
118+
LOGGER.info("create index failed for {}", category.getName(), e);
119+
}
120+
121+
if (providerName.equals(ProviderNames.DEFAULT)) {
122+
setTTLIndexInMockerCollection(category, database);
123+
}
124+
}
125+
}
126+
}
127+
128+
private void setTTLIndexInMockerCollection(MockCategoryType category,
129+
MongoDatabase mongoDatabase) {
130+
String categoryName = getCollectionName(category, ProviderNames.DEFAULT);
131+
MongoCollection<AREXMocker> collection = mongoDatabase.getCollection(categoryName,
132+
AREXMocker.class);
133+
Bson index = new Document(EXPIRATION_TIME_COLUMN_NAME, 1);
134+
IndexOptions indexOptions = new IndexOptions().expireAfter(0L, TimeUnit.SECONDS);
135+
indexOptions.background(true);
136+
try {
137+
collection.createIndex(index, indexOptions);
138+
} catch (MongoCommandException e) {
139+
// ignore
140+
collection.dropIndex(index);
141+
collection.createIndex(index, indexOptions);
142+
}
143+
}
144+
145+
private String getCollectionName(MockCategoryType category, String providerName) {
146+
return providerName + category.getName() + COLLECTION_SUFFIX;
147+
}
148+
149+
private void setIndexByEnum(MongoCollectionIndexConfigEnum indexConfigEnum, MongoDatabase mongoDatabase) {
150+
MongoCollection<Document> collection = mongoDatabase.getCollection(
151+
indexConfigEnum.getCollectionName());
152+
153+
ListIndexesIterable<Document> existedIndexes = collection.listIndexes();
154+
155+
List<Pair<Document, IndexOptions>> toAddIndexes = new ArrayList<>();
156+
157+
indexConfigEnum.getIndexConfigs().forEach(indexConfig -> {
158+
List<FieldConfig> fieldConfigs = indexConfig.getFieldConfigs();
159+
Document index = new Document();
160+
for (FieldConfig fieldConfig : fieldConfigs) {
161+
index.append(fieldConfig.getFieldName(),
162+
fieldConfig.getAscending() != Boolean.FALSE ? 1 : -1);
163+
}
164+
IndexOptions indexOptions = new IndexOptions();
165+
if (indexConfig.getUnique() != null) {
166+
indexOptions.unique(indexConfig.getUnique());
167+
}
168+
if (indexConfig.getTtlIndexConfig() != null) {
169+
TtlIndexConfig ttlIndexConfig = indexConfig.getTtlIndexConfig();
170+
indexOptions.expireAfter(ttlIndexConfig.getExpireAfter(), ttlIndexConfig.getTimeUnit());
171+
}
172+
indexOptions.background(true);
173+
toAddIndexes.add(Pair.of(index, indexOptions));
174+
});
175+
176+
// add new indexes which not exist
177+
for (Pair<Document, IndexOptions> newIndex : toAddIndexes) {
178+
if (!isIndexExist(existedIndexes, newIndex.getLeft(), newIndex.getRight())) {
179+
try {
180+
collection.createIndex(newIndex.getLeft(), newIndex.getRight());
181+
} catch (MongoCommandException e) {
182+
collection.dropIndex(newIndex.getLeft());
183+
collection.createIndex(newIndex.getLeft(), newIndex.getRight());
184+
}
185+
}
186+
}
187+
188+
}
189+
190+
private boolean isIndexExist(Iterable<Document> existedIndexes, Document index,
191+
IndexOptions indexOptions) {
192+
for (Document oldIndex : existedIndexes) {
193+
if (isMatch(oldIndex, index, indexOptions)) {
194+
return true;
195+
}
196+
}
197+
return false;
198+
}
199+
200+
private boolean isMatch(Document oldIndex, Document newIndex, IndexOptions newIndexOptions) {
201+
Document key = (Document) oldIndex.get(KEY);
202+
if (!newIndex.equals(key)) {
203+
return false;
204+
}
205+
if (newIndexOptions == null) {
206+
return false;
207+
}
208+
if (!Objects.equals(newIndexOptions.isBackground(), oldIndex.getBoolean(BACKGROUND))) {
209+
return false;
210+
}
211+
if (!Objects.equals(newIndexOptions.isUnique(), oldIndex.getBoolean(UNIQUE))) {
212+
return false;
213+
}
214+
if (!Objects.equals(newIndexOptions.getExpireAfter(TimeUnit.SECONDS),
215+
oldIndex.getLong(EXPIRE_AFTER_SECONDS))) {
216+
return false;
217+
}
218+
return true;
219+
}
220+
}

‎arex-storage-web-api/src/main/java/com/arextest/storage/beans/IndexsSettingConfiguration.java

Lines changed: 0 additions & 155 deletions
This file was deleted.

0 commit comments

Comments
 (0)