Skip to content

Commit 5bf6b6d

Browse files
qqmyersqqmyers
authored
Solr: Try Soft Commit on Indexing (#10547)
* try soft commit * keep softcommit short to avoid delays in visibility * add test delay for autosoft, make hardcommit 30s like auto setting * add 1-2 second delays in tests for softAutocomplete at 1s * more sleeps * more delays * remove commented out deletes * more commented out code to remove * add 1 sec on failing tests * add missing perm reindex * change waiting * fix index object and add null check for unit test * remove test-specific null check * reindex linking dv * general solr release note * more fixes * revert change - was correct * another sleepforsearch * test adding explicit reindexing * avoid other uses of cache in test that looks for exact counts * Adding longer max sleep, add count param to sleep method * Revert "add missing perm reindex" This reverts commit 317038a.
1 parent 3934c3f commit 5bf6b6d

File tree

10 files changed

+66
-68
lines changed

10 files changed

+66
-68
lines changed

‎conf/solr/9.3.0/solrconfig.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@
290290
have some sort of hard autoCommit to limit the log size.
291291
-->
292292
<autoCommit>
293-
<maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
293+
<maxTime>${solr.autoCommit.maxTime:30000}</maxTime>
294294
<openSearcher>false</openSearcher>
295295
</autoCommit>
296296

@@ -301,7 +301,7 @@
301301
-->
302302

303303
<autoSoftCommit>
304-
<maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
304+
<maxTime>${solr.autoSoftCommit.maxTime:1000}</maxTime>
305305
</autoSoftCommit>
306306

307307
<!-- Update Related Event Listeners

‎doc/release-notes/10547-solr-updates.md

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Multiple improvements have ben made to they way Solr indexing and searching is done. Response times should be significantly improved. See the individual PRs in this release for details.

‎src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java

+7-27
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ public Future<String> indexDataverse(Dataverse dataverse, boolean processPaths)
312312
String status;
313313
try {
314314
if (dataverse.getId() != null) {
315-
solrClientService.getSolrClient().add(docs);
315+
solrClientService.getSolrClient().add(docs, COMMIT_WITHIN);
316316
} else {
317317
logger.info("WARNING: indexing of a dataverse with no id attempted");
318318
}
@@ -321,14 +321,6 @@ public Future<String> indexDataverse(Dataverse dataverse, boolean processPaths)
321321
logger.info(status);
322322
return new AsyncResult<>(status);
323323
}
324-
try {
325-
solrClientService.getSolrClient().commit();
326-
} catch (SolrServerException | IOException ex) {
327-
status = ex.toString();
328-
logger.info(status);
329-
return new AsyncResult<>(status);
330-
}
331-
332324
dvObjectService.updateContentIndexTime(dataverse);
333325
IndexResponse indexResponse = solrIndexService.indexPermissionsForOneDvObject(dataverse);
334326
String msg = "indexed dataverse " + dataverse.getId() + ":" + dataverse.getAlias() + ". Response from permission indexing: " + indexResponse.getMessage();
@@ -353,6 +345,7 @@ public void indexDatasetInNewTransaction(Long datasetId) { //Dataset dataset) {
353345
private static final Map<Long, Boolean> INDEXING_NOW = new ConcurrentHashMap<>();
354346
// semaphore for async indexing
355347
private static final Semaphore ASYNC_INDEX_SEMAPHORE = new Semaphore(JvmSettings.MAX_ASYNC_INDEXES.lookupOptional(Integer.class).orElse(4), true);
348+
static final int COMMIT_WITHIN = 30000; //Same as current autoHardIndex time
356349

357350
@Inject
358351
@Metric(name = "index_permit_wait_time", absolute = true, unit = MetricUnits.NANOSECONDS,
@@ -1535,8 +1528,7 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set<Long> d
15351528
final SolrInputDocuments docs = toSolrDocs(indexableDataset, datafilesInDraftVersion);
15361529

15371530
try {
1538-
solrClientService.getSolrClient().add(docs.getDocuments());
1539-
solrClientService.getSolrClient().commit();
1531+
solrClientService.getSolrClient().add(docs.getDocuments(), COMMIT_WITHIN);
15401532
} catch (SolrServerException | IOException ex) {
15411533
if (ex.getCause() instanceof SolrServerException) {
15421534
throw new SolrServerException(ex);
@@ -1788,8 +1780,7 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc
17881780

17891781
sid.removeField(SearchFields.SUBTREE);
17901782
sid.addField(SearchFields.SUBTREE, paths);
1791-
UpdateResponse addResponse = solrClientService.getSolrClient().add(sid);
1792-
UpdateResponse commitResponse = solrClientService.getSolrClient().commit();
1783+
UpdateResponse addResponse = solrClientService.getSolrClient().add(sid, COMMIT_WITHIN);
17931784
if (object.isInstanceofDataset()) {
17941785
for (DataFile df : dataset.getFiles()) {
17951786
solrQuery.setQuery(SearchUtil.constructQuery(SearchFields.ENTITY_ID, df.getId().toString()));
@@ -1802,8 +1793,7 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc
18021793
}
18031794
sid.removeField(SearchFields.SUBTREE);
18041795
sid.addField(SearchFields.SUBTREE, paths);
1805-
addResponse = solrClientService.getSolrClient().add(sid);
1806-
commitResponse = solrClientService.getSolrClient().commit();
1796+
addResponse = solrClientService.getSolrClient().add(sid, COMMIT_WITHIN);
18071797
}
18081798
}
18091799
}
@@ -1845,12 +1835,7 @@ public String delete(Dataverse doomed) {
18451835
logger.fine("deleting Solr document for dataverse " + doomed.getId());
18461836
UpdateResponse updateResponse;
18471837
try {
1848-
updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId());
1849-
} catch (SolrServerException | IOException ex) {
1850-
return ex.toString();
1851-
}
1852-
try {
1853-
solrClientService.getSolrClient().commit();
1838+
updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId(), COMMIT_WITHIN);
18541839
} catch (SolrServerException | IOException ex) {
18551840
return ex.toString();
18561841
}
@@ -1870,12 +1855,7 @@ public String removeSolrDocFromIndex(String doomed) {
18701855
logger.fine("deleting Solr document: " + doomed);
18711856
UpdateResponse updateResponse;
18721857
try {
1873-
updateResponse = solrClientService.getSolrClient().deleteById(doomed);
1874-
} catch (SolrServerException | IOException ex) {
1875-
return ex.toString();
1876-
}
1877-
try {
1878-
solrClientService.getSolrClient().commit();
1858+
updateResponse = solrClientService.getSolrClient().deleteById(doomed, COMMIT_WITHIN);
18791859
} catch (SolrServerException | IOException ex) {
18801860
return ex.toString();
18811861
}

‎src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java

+3-10
Original file line numberDiff line numberDiff line change
@@ -356,8 +356,7 @@ private void persistToSolr(Collection<SolrInputDocument> docs) throws SolrServer
356356
/**
357357
* @todo Do something with these responses from Solr.
358358
*/
359-
UpdateResponse addResponse = solrClientService.getSolrClient().add(docs);
360-
UpdateResponse commitResponse = solrClientService.getSolrClient().commit();
359+
UpdateResponse addResponse = solrClientService.getSolrClient().add(docs, IndexServiceBean.COMMIT_WITHIN);
361360
}
362361

363362
public IndexResponse indexPermissionsOnSelfAndChildren(long definitionPointId) {
@@ -497,26 +496,20 @@ public IndexResponse deleteMultipleSolrIds(List<String> solrIdsToDelete) {
497496
return new IndexResponse("nothing to delete");
498497
}
499498
try {
500-
solrClientService.getSolrClient().deleteById(solrIdsToDelete);
499+
solrClientService.getSolrClient().deleteById(solrIdsToDelete, IndexServiceBean.COMMIT_WITHIN);
501500
} catch (SolrServerException | IOException ex) {
502501
/**
503502
* @todo mark these for re-deletion
504503
*/
505504
return new IndexResponse("problem deleting the following documents from Solr: " + solrIdsToDelete);
506505
}
507-
try {
508-
solrClientService.getSolrClient().commit();
509-
} catch (SolrServerException | IOException ex) {
510-
return new IndexResponse("problem committing deletion of the following documents from Solr: " + solrIdsToDelete);
511-
}
512506
return new IndexResponse("no known problem deleting the following documents from Solr:" + solrIdsToDelete);
513507
}
514508

515509
public JsonObjectBuilder deleteAllFromSolrAndResetIndexTimes() throws SolrServerException, IOException {
516510
JsonObjectBuilder response = Json.createObjectBuilder();
517511
logger.info("attempting to delete all Solr documents before a complete re-index");
518-
solrClientService.getSolrClient().deleteByQuery("*:*");
519-
solrClientService.getSolrClient().commit();
512+
solrClientService.getSolrClient().deleteByQuery("*:*", IndexServiceBean.COMMIT_WITHIN);
520513
int numRowsAffected = dvObjectService.clearAllIndexTimes();
521514
response.add(numRowsClearedByClearAllIndexTimes, numRowsAffected);
522515
response.add(messageString, "Solr index and database index timestamps cleared.");

‎src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ public void testMoveDataverse() {
438438
while (checkIndex) {
439439
try {
440440
try {
441-
Thread.sleep(4000);
441+
Thread.sleep(6000);
442442
} catch (InterruptedException ex) {
443443
}
444444
Response search = UtilIT.search("id:dataverse_" + dataverseId + "&subtree=" + dataverseAlias2, apiToken);

‎src/test/java/edu/harvard/iq/dataverse/api/LinkIT.java

+6
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import static jakarta.ws.rs.core.Response.Status.FORBIDDEN;
1010
import static jakarta.ws.rs.core.Response.Status.OK;
1111
import static org.hamcrest.CoreMatchers.equalTo;
12+
import static org.junit.jupiter.api.Assertions.assertTrue;
13+
1214
import org.junit.jupiter.api.BeforeAll;
1315
import org.junit.jupiter.api.Test;
1416

@@ -163,6 +165,8 @@ public void testDeepLinks() {
163165
.statusCode(OK.getStatusCode())
164166
.body("data.message", equalTo("Dataverse " + level1a + " linked successfully to " + level1b));
165167

168+
assertTrue(UtilIT.sleepForSearch("*", apiToken, "&subtree="+level1b, 1, UtilIT.GENERAL_LONG_DURATION), "Zero counts in level1b");
169+
166170
Response searchLevel1toLevel1 = UtilIT.search("*", apiToken, "&subtree=" + level1b);
167171
searchLevel1toLevel1.prettyPrint();
168172
searchLevel1toLevel1.then().assertThat()
@@ -184,6 +188,8 @@ public void testDeepLinks() {
184188
.statusCode(OK.getStatusCode())
185189
.body("data.message", equalTo("Dataverse " + level2a + " linked successfully to " + level2b));
186190

191+
assertTrue(UtilIT.sleepForSearch("*", apiToken, "&subtree=" + level2b, 1, UtilIT.GENERAL_LONG_DURATION), "Never found linked dataverse: " + level2b);
192+
187193
Response searchLevel2toLevel2 = UtilIT.search("*", apiToken, "&subtree=" + level2b);
188194
searchLevel2toLevel2.prettyPrint();
189195
searchLevel2toLevel2.then().assertThat()

‎src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java

+12-25
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ public void testSearchPermisions() throws InterruptedException {
105105
assertEquals(200, grantUser2AccessOnDataset.getStatusCode());
106106

107107
String searchPart = "id:dataset_" + datasetId1 + "_draft";
108-
assertTrue(UtilIT.sleepForSearch(searchPart, apiToken2, "", UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if search exceeds max duration " + searchPart);
108+
assertTrue(UtilIT.sleepForSearch(searchPart, apiToken2, "", 1, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if search exceeds max duration " + searchPart);
109109

110110
Response shouldBeVisibleToUser2 = UtilIT.search("id:dataset_" + datasetId1 + "_draft", apiToken2);
111111
shouldBeVisibleToUser2.prettyPrint();
@@ -793,14 +793,9 @@ public void testNestedSubtree() {
793793
Response createDataverseResponse2 = UtilIT.createSubDataverse("subDV" + UtilIT.getRandomIdentifier(), null, apiToken, dataverseAlias);
794794
createDataverseResponse2.prettyPrint();
795795
String dataverseAlias2 = UtilIT.getAliasFromResponse(createDataverseResponse2);
796-
796+
797797
String searchPart = "*";
798-
799-
Response searchUnpublishedSubtree = UtilIT.search(searchPart, apiToken, "&subtree="+dataverseAlias);
800-
searchUnpublishedSubtree.prettyPrint();
801-
searchUnpublishedSubtree.then().assertThat()
802-
.statusCode(OK.getStatusCode())
803-
.body("data.total_count", CoreMatchers.equalTo(1));
798+
assertTrue(UtilIT.sleepForSearch(searchPart, apiToken, "&subtree=" + dataverseAlias, 1, UtilIT.GENERAL_LONG_DURATION), "Missing subDV");
804799

805800
Response searchUnpublishedSubtree2 = UtilIT.search(searchPart, apiToken, "&subtree="+dataverseAlias2);
806801
searchUnpublishedSubtree2.prettyPrint();
@@ -863,18 +858,8 @@ public void testNestedSubtree() {
863858
publishDataset.then().assertThat()
864859
.statusCode(OK.getStatusCode());
865860
UtilIT.sleepForReindex(datasetPid, apiToken, 5);
866-
Response searchPublishedSubtreeWDS = UtilIT.search(searchPart, apiToken, "&subtree="+dataverseAlias);
867-
searchPublishedSubtreeWDS.prettyPrint();
868-
searchPublishedSubtreeWDS.then().assertThat()
869-
.statusCode(OK.getStatusCode())
870-
.body("data.total_count", CoreMatchers.equalTo(2));
871-
872-
Response searchPublishedSubtreeWDS2 = UtilIT.search(searchPart, apiToken, "&subtree="+dataverseAlias2);
873-
searchPublishedSubtreeWDS2.prettyPrint();
874-
searchPublishedSubtreeWDS2.then().assertThat()
875-
.statusCode(OK.getStatusCode())
876-
.body("data.total_count", CoreMatchers.equalTo(1));
877-
861+
assertTrue(UtilIT.sleepForSearch(searchPart, apiToken, "&subtree=" + dataverseAlias, 2, UtilIT.GENERAL_LONG_DURATION), "Did not find 2 children");
862+
assertTrue(UtilIT.sleepForSearch(searchPart, apiToken, "&subtree=" + dataverseAlias2, 1, UtilIT.GENERAL_LONG_DURATION), "Did not find 1 child");
878863
}
879864

880865
//If this test fails it'll fail inconsistently as it tests underlying async role code
@@ -906,16 +891,16 @@ public void testCuratorCardDataversePopulation() throws InterruptedException {
906891
String subDataverseAlias = "dv" + UtilIT.getRandomIdentifier();
907892
Response createSubDataverseResponse = UtilIT.createSubDataverse(subDataverseAlias, null, apiTokenSuper, parentDataverseAlias);
908893
createSubDataverseResponse.prettyPrint();
909-
//UtilIT.getAliasFromResponse(createSubDataverseResponse);
910-
894+
911895
Response grantRoleOnDataverseResponse = UtilIT.grantRoleOnDataverse(subDataverseAlias, "curator", "@" + username, apiTokenSuper);
912896
grantRoleOnDataverseResponse.then().assertThat()
913897
.statusCode(OK.getStatusCode());
914-
898+
915899
String searchPart = "*";
916900

901+
assertTrue(UtilIT.sleepForSearch(searchPart, apiToken, "&subtree="+parentDataverseAlias, 1, UtilIT.GENERAL_LONG_DURATION), "Failed test if search exceeds max duration " + searchPart);
902+
917903
Response searchPublishedSubtreeSuper = UtilIT.search(searchPart, apiTokenSuper, "&subtree="+parentDataverseAlias);
918-
assertTrue(UtilIT.sleepForSearch(searchPart, apiToken, "&subtree="+parentDataverseAlias, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if search exceeds max duration " + searchPart);
919904
searchPublishedSubtreeSuper.prettyPrint();
920905
searchPublishedSubtreeSuper.then().assertThat()
921906
.statusCode(OK.getStatusCode())
@@ -968,7 +953,7 @@ public void testSubtreePermissions() {
968953
.statusCode(OK.getStatusCode());
969954

970955
// Wait a little while for the index to pick up the datasets, otherwise timing issue with searching for it.
971-
UtilIT.sleepForReindex(datasetId2.toString(), apiToken, 2);
956+
UtilIT.sleepForReindex(datasetId2.toString(), apiToken, 3);
972957

973958
String identifier = JsonPath.from(datasetAsJson.getBody().asString()).getString("data.identifier");
974959
String identifier2 = JsonPath.from(datasetAsJson2.getBody().asString()).getString("data.identifier");
@@ -1077,6 +1062,8 @@ public void testSubtreePermissions() {
10771062
.statusCode(OK.getStatusCode())
10781063
.body("data.total_count", CoreMatchers.equalTo(2));
10791064

1065+
assertTrue(UtilIT.sleepForSearch(searchPart, null, "&subtree=" + dataverseAlias2, 1, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Missing dataset w/no apiKey");
1066+
10801067
Response searchPublishedSubtreesNoAPI = UtilIT.search(searchPart, null, "&subtree="+dataverseAlias+"&subtree="+dataverseAlias2);
10811068
searchPublishedSubtreesNoAPI.prettyPrint();
10821069
searchPublishedSubtreesNoAPI.then().assertThat()

‎src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java

+16-3
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ public class UtilIT {
6464
private static final String EMPTY_STRING = "";
6565
public static final int MAXIMUM_INGEST_LOCK_DURATION = 15;
6666
public static final int MAXIMUM_PUBLISH_LOCK_DURATION = 20;
67+
public static final int GENERAL_LONG_DURATION = 45; //Useful when multiple adds/publishes, etc/ all get done in sequence
6768
public static final int MAXIMUM_IMPORT_DURATION = 1;
6869

6970
private static SwordConfigurationImpl swordConfiguration = new SwordConfigurationImpl();
@@ -2845,6 +2846,13 @@ static boolean sleepForReindex(String idOrPersistentId, String apiToken, int dur
28452846
i = repeats + 1;
28462847
}
28472848
} while ((i <= repeats) && stale);
2849+
try {
2850+
Thread.sleep(1000); //Current autoSoftIndexTime - which adds a delay to when the new docs are visible
2851+
i++;
2852+
} catch (InterruptedException ex) {
2853+
Logger.getLogger(UtilIT.class.getName()).log(Level.SEVERE, null, ex);
2854+
i = repeats + 1;
2855+
}
28482856
System.out.println("Waited " + (i * (sleepStep / 1000.0)) + " seconds");
28492857
return i <= repeats;
28502858

@@ -2900,10 +2908,15 @@ static Boolean sleepForDeadlock(int duration) {
29002908

29012909
//Helper function that returns true if a given search returns a non-zero response within a fixed time limit
29022910
// a given duration returns false if still zero results after given duration
2903-
static Boolean sleepForSearch(String searchPart, String apiToken, String subTree, int duration) {
2911+
static Boolean sleepForSearch(String searchPart, String apiToken, String subTree, int count, int duration) {
29042912

29052913

29062914
Response searchResponse = UtilIT.search(searchPart, apiToken, subTree);
2915+
//Leave early if search isn't working
2916+
if(searchResponse.statusCode()!=200) {
2917+
logger.warning("Non-200 status in sleepForSearch: " + searchResponse.statusCode());
2918+
return false;
2919+
}
29072920
int i = 0;
29082921
do {
29092922
try {
@@ -2916,8 +2929,8 @@ static Boolean sleepForSearch(String searchPart, String apiToken, String subTre
29162929
} catch (InterruptedException ex) {
29172930
Logger.getLogger(UtilIT.class.getName()).log(Level.SEVERE, null, ex);
29182931
}
2919-
} while (UtilIT.getSearchCountFromResponse(searchResponse) == 0);
2920-
2932+
} while (UtilIT.getSearchCountFromResponse(searchResponse) != count);
2933+
logger.info("Waited " + i + " seconds in sleepForSearch");
29212934
return i <= duration;
29222935

29232936
}

‎src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreatePrivateUrlCommandTest.java

+16
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import edu.harvard.iq.dataverse.Dataset;
44
import edu.harvard.iq.dataverse.DatasetVersion;
5+
import edu.harvard.iq.dataverse.Dataverse;
56
import edu.harvard.iq.dataverse.DataverseRoleServiceBean;
67
import edu.harvard.iq.dataverse.DvObject;
78
import edu.harvard.iq.dataverse.RoleAssignment;
@@ -13,9 +14,14 @@
1314
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
1415
import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
1516
import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
17+
import edu.harvard.iq.dataverse.search.IndexResponse;
18+
import edu.harvard.iq.dataverse.search.IndexServiceBean;
19+
import edu.harvard.iq.dataverse.search.SolrIndexServiceBean;
1620
import edu.harvard.iq.dataverse.util.SystemConfig;
1721
import java.util.ArrayList;
1822
import java.util.List;
23+
import java.util.concurrent.Future;
24+
1925
import org.junit.jupiter.api.BeforeEach;
2026
import org.junit.jupiter.api.Test;
2127

@@ -95,6 +101,16 @@ public String getDataverseSiteUrl() {
95101
};
96102

97103
}
104+
105+
@Override
106+
public SolrIndexServiceBean solrIndex() {
107+
return new SolrIndexServiceBean(){
108+
@Override
109+
public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) {
110+
return null;
111+
}
112+
};
113+
}
98114

99115
}
100116
);

0 commit comments

Comments
 (0)