avoid expensive Solr join when guest users search (affect IP Groups) #10554

Author: pdurbin

doc/release-notes/10554-avoid-solr-join-guest.md

@@ -0,0 +1,3 @@
+An experimental feature flag called "avoid-expensive-solr-join" has been added to change the way Solr queries are constructed for guest (unauthenticated) users. It is hoped that it will help with performance, reducing load on Solr.
+
+From a search perspective, it disables IP Groups (collections, datasets, and files will not be discoverable) but it removes an expensive Solr join for the most common users, which are guests. After turning on this feature, you must perform a full reindex.

doc/sphinx-guides/source/admin/ip-groups.rst

@@ -41,3 +41,8 @@ It is not recommended to delete an IP Group that has been assigned roles. If you
 To delete an IP Group with an alias of "ipGroup1", use the curl command below:
 
 ``curl -X DELETE http://localhost:8080/api/admin/groups/ip/ipGroup1``
+
+Related Settings
+----------------
+
+Be aware that enabling the feature flag ``avoid-expensive-solr-join`` will effectively prevent collection, datasets, and files from being found by members of IP Groups when searching, rendering IP Groups much less useful. See :ref:`feature-flags` in the Installation Guide for details.

doc/sphinx-guides/source/developers/performance.rst

@@ -118,6 +118,10 @@ Solr
 
 While in the past Solr performance hasn't been much of a concern, in recent years we've noticed performance problems when Harvard Dataverse is under load. Improvements were made in `PR #10050 <https://github.com/IQSS/dataverse/pull/10050>`_, for example.
 
+We are tracking performance problems in `#10469 <https://github.com/IQSS/dataverse/issues/10469>`_.
+
+In a meeting with a Solr expert on 2024-05-10 we were advised to avoid joins as much as possible. (It was acknowledged that many Solr users make use of joins because they have to, like we do, to keep some documents private.) Toward that end we have added a feature flag called ``avoid-expensive-solr-join`` as explained under :ref:`feature-flags`.
+
 Datasets with Large Numbers of Files or Versions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/sphinx-guides/source/installation/config.rst

@@ -3248,6 +3248,9 @@ please find all known feature flags below. Any of these flags can be activated u
     * - api-session-auth
       - Enables API authentication via session cookie (JSESSIONID). **Caution: Enabling this feature flag exposes the installation to CSRF risks!** We expect this feature flag to be temporary (only used by frontend developers, see `#9063 <https://github.com/IQSS/dataverse/issues/9063>`_) and for the feature to be removed in the future.
       - ``Off``
+    * - avoid-expensive-solr-join
+      - Changes the way Solr queries are constructed for guest (unauthenticated) users. From a search perspective, it disables :doc:`IP Groups </admin/ip-groups>` (collections, datasets, and files will not be discoverable) but it removes an expensive Solr join for the most common users, which are guests, to help improve overall performance. After turning on this feature, you must perform a full reindex. See :doc:`/admin/solr-search-index`.
+      - ``Off``
 
 **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable
 ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development </container/dev-usage>`, you can set them in the `docker compose <https://docs.docker.com/compose/environment-variables/set-environment-variables/>`_ file.

src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java

@@ -12,6 +12,7 @@
 import edu.harvard.iq.dataverse.datavariable.VariableMetadataUtil;
 import edu.harvard.iq.dataverse.datavariable.VariableServiceBean;
 import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
+import edu.harvard.iq.dataverse.settings.FeatureFlags;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.FileUtil;
@@ -214,6 +215,9 @@ public Future<String> indexDataverse(Dataverse dataverse, boolean processPaths)
         solrInputDocument.addField(SearchFields.DATAVERSE_CATEGORY, dataverse.getIndexableCategoryName());
         if (dataverse.isReleased()) {
             solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, PUBLISHED_STRING);
+            if (FeatureFlags.AVOID_EXPENSIVE_SOLR_JOIN.enabled()) {
+                solrInputDocument.addField(SearchFields.PUBLIC_OBJECT, true);
+            }
             solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE, dataverse.getPublicationDate());
         } else {
             solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, UNPUBLISHED_STRING);
@@ -887,6 +891,9 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
 
         if (state.equals(indexableDataset.getDatasetState().PUBLISHED)) {
             solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, PUBLISHED_STRING);
+            if (FeatureFlags.AVOID_EXPENSIVE_SOLR_JOIN.enabled()) {
+                solrInputDocument.addField(SearchFields.PUBLIC_OBJECT, true);
+            }
             // solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE,
             // dataset.getPublicationDate());
         } else if (state.equals(indexableDataset.getDatasetState().WORKING_COPY)) {
@@ -1400,6 +1407,9 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                     if (indexableDataset.getDatasetState().equals(indexableDataset.getDatasetState().PUBLISHED)) {
                         fileSolrDocId = solrDocIdentifierFile + fileEntityId;
                         datafileSolrInputDocument.addField(SearchFields.PUBLICATION_STATUS, PUBLISHED_STRING);
+                        if (FeatureFlags.AVOID_EXPENSIVE_SOLR_JOIN.enabled()) {
+                            solrInputDocument.addField(SearchFields.PUBLIC_OBJECT, true);
+                        }
                         // datafileSolrInputDocument.addField(SearchFields.PERMS, publicGroupString);
                         addDatasetReleaseDateToSolrDoc(datafileSolrInputDocument, dataset);
                         // has this published file been deleted from the current draft version? 

src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java

@@ -217,6 +217,15 @@ public class SearchFields {
     public static final String DEFINITION_POINT_DVOBJECT_ID = "definitionPointDvObjectId";
     public static final String DISCOVERABLE_BY = "discoverableBy";
 
+    /**
+     * publicObject_b is an experimental field tied to the
+     * avoid-expensive-solr-join feature flag. Rather than discoverableBy which
+     * is a field on permission documents, publicObject_b is a field on content
+     * documents (dvObjects). By indexing publicObject_b=true, we can let guests
+     * search on it, avoiding an expensive join for those (common) users.
+     */
+    public static final String PUBLIC_OBJECT = "publicObject_b";
+
     /**
      * i.e. "Unpublished", "Draft" (multivalued)
      */

src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java

@@ -16,6 +16,7 @@
 import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.settings.FeatureFlags;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import java.io.IOException;
@@ -1006,6 +1007,14 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
         // Yes, see if GuestUser is part of any groups such as IP Groups.
         // ----------------------------------------------------
         if (user instanceof GuestUser) {
+            if (FeatureFlags.AVOID_EXPENSIVE_SOLR_JOIN.enabled()) {
+                /**
+                 * Instead of doing an expensive join, narrow down to only
+                 * public objects. This field is indexed on the content document
+                 * itself, rather than a permission document.
+                 */
+                return SearchFields.PUBLIC_OBJECT + ":" + true;
+            }
             String groupsFromProviders = "";
             Set<Group> groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest));
             StringBuilder sb = new StringBuilder();

src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java

@@ -36,6 +36,15 @@ public enum FeatureFlags {
      * @since Dataverse @TODO:
      */
     API_BEARER_AUTH("api-bearer-auth"),
+    /**
+     * For Guest users, don't use a join when searching Solr. Disables the IP
+     * Groups feature from a search perspective. Requires a reindex.
+     *
+     * @apiNote Raise flag by setting
+     * "dataverse.feature.avoid-expensive-solr-join"
+     * @since Dataverse @TODO:
+     */
+    AVOID_EXPENSIVE_SOLR_JOIN("avoid-expensive-solr-join"),
     ;
 
     final String flag;