[main] Move system indices migration to migrate plugin (elastic#125437)

* [main] Move system indices migration to migrate plugin

It seems the best way to fix elastic#122949 is to use existing data stream reindex API. However, this API is located in the migrate x-pack plugin. This commit moves the system indices migration logic (REST handlers, transport actions, and task) to the migrate plugin.

Port of elastic#123551

* [CI] Auto commit changes from spotless

* Fix compilation

* Fix tests

* Fix test

---------

Co-authored-by: elasticsearchmachine <[email protected]>

Author: alexey-ivanov-es

modules/reindex/build.gradle

@@ -44,9 +44,6 @@ dependencies {
   clusterModules project(':modules:lang-painless')
   clusterModules project(':modules:parent-join')
   clusterModules project(":modules:rest-root")
-
-  internalClusterTestImplementation project(':modules:lang-painless')
-  internalClusterTestImplementation project(':modules:lang-painless:spi')
 }
 
 restResources {

modules/reindex/src/main/plugin-metadata/plugin-security.policy

@@ -10,9 +10,6 @@
 grant {
    // reindex opens socket connections using the rest client
    permission java.net.SocketPermission "*", "connect";
-
-  // needed for Painless to generate runtime classes
-  permission java.lang.RuntimePermission "createClassLoader";
 };
 
 grant codeBase "${codebase.elasticsearch-rest-client}" {

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java

@@ -11,9 +11,9 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 
-import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.test.XContentTestUtils;
 import org.junit.BeforeClass;
 
@@ -112,7 +112,7 @@ public void testGetFeatureUpgradeStatus() throws Exception {
                 // for the next major version upgrade (see e.g. #93666). Trying to express this with features may be problematic, so we
                 // want to keep using versions here. We also assume that for non-semantic version migrations are not required.
                 boolean migrationNeeded = parseLegacyVersion(getOldClusterVersion()).map(
-                    v -> v.before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION)
+                    v -> v.before(SystemIndices.NO_UPGRADE_REQUIRED_VERSION)
                 ).orElse(false);
                 if (migrationNeeded) {
                     assertThat(feature.get("migration_status"), equalTo("MIGRATION_NEEDED"));

qa/system-indices/build.gradle

@@ -17,3 +17,8 @@ esplugin {
   licenseFile = layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile
   noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile
 }
+
+dependencies {
+  clusterModules project(':modules:ingest-common')
+  clusterModules project(':x-pack:plugin:migrate')
+}

qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/AbstractSystemIndicesIT.java

@@ -25,6 +25,8 @@ public abstract class AbstractSystemIndicesIT extends ESRestTestCase {
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .plugin("system-indices-qa")
+        .module("ingest-common")
+        .module("x-pack-migrate")
         .setting("xpack.security.enabled", "true")
         .setting("xpack.security.autoconfiguration.enabled", "false")
         .user("rest_user", "rest-user-password")

server/src/main/java/module-info.java

@@ -62,7 +62,6 @@
     exports org.elasticsearch.action.admin.cluster.coordination;
     exports org.elasticsearch.action.admin.cluster.desirednodes;
     exports org.elasticsearch.action.admin.cluster.health;
-    exports org.elasticsearch.action.admin.cluster.migration;
     exports org.elasticsearch.action.admin.cluster.node.capabilities;
     exports org.elasticsearch.action.admin.cluster.node.hotthreads;
     exports org.elasticsearch.action.admin.cluster.node.info;
@@ -382,7 +381,6 @@
     exports org.elasticsearch.tasks;
     exports org.elasticsearch.threadpool;
     exports org.elasticsearch.transport;
-    exports org.elasticsearch.upgrades;
     exports org.elasticsearch.usage;
     exports org.elasticsearch.watcher;
 

server/src/main/java/org/elasticsearch/action/ActionModule.java

@@ -26,10 +26,6 @@
 import org.elasticsearch.action.admin.cluster.desirednodes.TransportUpdateDesiredNodesAction;
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction;
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
-import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction;
-import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction;
-import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
-import org.elasticsearch.action.admin.cluster.migration.TransportPostFeatureUpgradeAction;
 import org.elasticsearch.action.admin.cluster.node.capabilities.TransportNodesCapabilitiesAction;
 import org.elasticsearch.action.admin.cluster.node.features.TransportNodesFeaturesAction;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction;
@@ -286,7 +282,6 @@
 import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredBalanceAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredNodesAction;
-import org.elasticsearch.rest.action.admin.cluster.RestGetFeatureUpgradeStatusAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction;
@@ -300,7 +295,6 @@
 import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction;
-import org.elasticsearch.rest.action.admin.cluster.RestPostFeatureUpgradeAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPrevalidateNodeRemovalAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
@@ -681,10 +675,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(TransportSnapshotsStatusAction.TYPE, TransportSnapshotsStatusAction.class);
         actions.register(SnapshottableFeaturesAction.INSTANCE, TransportSnapshottableFeaturesAction.class);
         actions.register(ResetFeatureStateAction.INSTANCE, TransportResetFeatureStateAction.class);
-        actions.register(GetFeatureUpgradeStatusAction.INSTANCE, TransportGetFeatureUpgradeStatusAction.class);
-        actions.register(PostFeatureUpgradeAction.INSTANCE, TransportPostFeatureUpgradeAction.class);
         actions.register(TransportGetShardSnapshotAction.TYPE, TransportGetShardSnapshotAction.class);
-
         actions.register(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class);
         actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class);
         actions.register(TransportIndicesShardStoresAction.TYPE, TransportIndicesShardStoresAction.class);
@@ -882,8 +873,6 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
         registerHandler.accept(new RestSnapshotsStatusAction());
         registerHandler.accept(new RestSnapshottableFeaturesAction());
         registerHandler.accept(new RestResetFeatureStateAction());
-        registerHandler.accept(new RestGetFeatureUpgradeStatusAction());
-        registerHandler.accept(new RestPostFeatureUpgradeAction());
         registerHandler.accept(new RestGetIndicesAction());
         registerHandler.accept(new RestIndicesStatsAction());
         registerHandler.accept(new RestIndicesSegmentsAction());

server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java

@@ -148,7 +148,7 @@ protected void masterOperation(
                 listener.onFailure(new IllegalStateException(message));
                 return;
             }
-            updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor, projectId);
+            updateRequest = buildManagedSystemIndexUpdateRequest(request, cause, descriptor, projectId);
         } else {
             updateRequest = buildUpdateRequest(request, cause, indexName, resolvedAt, projectId);
         }
@@ -181,31 +181,55 @@ private CreateIndexClusterStateUpdateRequest buildUpdateRequest(
             .waitForActiveShards(request.waitForActiveShards());
     }
 
-    private static CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateRequest(
+    private static CreateIndexClusterStateUpdateRequest buildManagedSystemIndexUpdateRequest(
         CreateIndexRequest request,
         String cause,
         SystemIndexDescriptor descriptor,
         ProjectId projectId
     ) {
-        final Settings settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY);
+        boolean indexMigrationInProgress = cause.equals(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE)
+            && request.index().endsWith(SystemIndices.UPGRADED_INDEX_SUFFIX);
 
+        final Settings settings;
+        final String mappings;
         final Set<Alias> aliases;
-        if (descriptor.getAliasName() == null) {
+        final String indexName;
+
+        // if we are migrating a system index to a new index, we use settings/mappings/index name from the request,
+        // since it was created by SystemIndexMigrator
+        if (indexMigrationInProgress) {
+            settings = request.settings();
+            mappings = request.mappings();
+            indexName = request.index();
+            // we will update alias later on
             aliases = Set.of();
         } else {
-            aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true));
-        }
+            settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY);
+            mappings = descriptor.getMappings();
+
+            if (descriptor.getAliasName() == null) {
+                aliases = Set.of();
+            } else {
+                aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true));
+            }
 
-        // Throw an error if we are trying to directly create a system index other than the primary system index (or the alias)
-        if (request.index().equals(descriptor.getPrimaryIndex()) == false && request.index().equals(descriptor.getAliasName()) == false) {
-            throw new IllegalArgumentException(
-                "Cannot create system index with name " + request.index() + "; descriptor primary index is " + descriptor.getPrimaryIndex()
-            );
+            // Throw an error if we are trying to directly create a system index other
+            // than the primary system index (or the alias, or we are migrating the index)
+            if (request.index().equals(descriptor.getPrimaryIndex()) == false
+                && request.index().equals(descriptor.getAliasName()) == false) {
+                throw new IllegalArgumentException(
+                    "Cannot create system index with name "
+                        + request.index()
+                        + "; descriptor primary index is "
+                        + descriptor.getPrimaryIndex()
+                );
+            }
+            indexName = descriptor.getPrimaryIndex();
         }
 
-        return new CreateIndexClusterStateUpdateRequest(cause, projectId, descriptor.getPrimaryIndex(), request.index()).aliases(aliases)
+        return new CreateIndexClusterStateUpdateRequest(cause, projectId, indexName, request.index()).aliases(aliases)
             .waitForActiveShards(ActiveShardCount.ALL)
-            .mappings(descriptor.getMappings())
+            .mappings(mappings)
             .settings(settings);
     }
 }

server/src/main/java/org/elasticsearch/cluster/ClusterModule.java

@@ -89,7 +89,6 @@
 import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.upgrades.FeatureMigrationResults;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
 
@@ -257,7 +256,6 @@ public static List<Entry> getNamedWriteables() {
             ComposableIndexTemplateMetadata::readDiffFrom
         );
         registerProjectCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);
-        registerProjectCustom(entries, FeatureMigrationResults.TYPE, FeatureMigrationResults::new, FeatureMigrationResults::readDiffFrom);
         registerMetadataCustom(entries, NodesShutdownMetadata.TYPE, NodesShutdownMetadata::new, NodesShutdownMetadata::readDiffFrom);
         registerMetadataCustom(entries, DesiredNodesMetadata.TYPE, DesiredNodesMetadata::new, DesiredNodesMetadata::readDiffFrom);
         registerProjectCustom(

server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java

@@ -13,7 +13,6 @@
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
-import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
 import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -148,7 +147,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
 
     /**
      * An optional reindexing script to use when migrating an index created
-     * before {@link TransportGetFeatureUpgradeStatusAction#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
+     * before {@link SystemIndices#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
      * This script can be used to modify documents before they are added to the new index.
      * For example, it can be used to remove deprecated fields from the index.
      * <br>

server/src/main/java/org/elasticsearch/indices/SystemIndices.java

@@ -15,8 +15,8 @@
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
 import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse.ResetFeatureStateStatus;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
@@ -36,7 +36,10 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.snapshots.SnapshotsService;
 
@@ -111,7 +114,16 @@
 public class SystemIndices {
     public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed";
     public static final String EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_external_system_index_access_origin";
-    private static final int UPGRADED_TO_VERSION = TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION.major + 1;
+
+    /**
+     * These versions should be set to current major and current major's index version
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA)
+    public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_9_0_0;
+    public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.UPGRADE_TO_LUCENE_10_0_0;
+
+    public static final String MIGRATE_SYSTEM_INDEX_CAUSE = "migrate-system-index";
+    private static final int UPGRADED_TO_VERSION = NO_UPGRADE_REQUIRED_VERSION.major + 1;
     public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-" + UPGRADED_TO_VERSION;
 
     private static final Automaton EMPTY = Automata.makeEmpty();
@@ -346,6 +358,7 @@ public ExecutorSelector getExecutorSelector() {
 
     /**
      * Finds a single matching {@link SystemIndexDescriptor}, if any, for the given index name.
+     * Does not take into account system data streams and their backing indices.
      * @param name the name of the index
      * @return The matching {@link SystemIndexDescriptor} or {@code null} if no descriptor is found
      */
@@ -354,7 +367,7 @@ public ExecutorSelector getExecutorSelector() {
     }
 
     @Nullable
-    static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) {
+    private static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) {
         SystemIndexDescriptor matchingDescriptor = null;
         for (SystemIndexDescriptor systemIndexDescriptor : indexDescriptors) {
             if (systemIndexDescriptor.matchesIndexPattern(name)) {

server/src/main/java/org/elasticsearch/node/NodeConstruction.java

@@ -219,7 +219,6 @@
 import org.elasticsearch.threadpool.internal.BuiltInExecutorBuilders;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.upgrades.SystemIndexMigrationExecutor;
 import org.elasticsearch.usage.UsageService;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -577,8 +576,7 @@ private void createClientAndRegistries(Settings settings, ThreadPool threadPool,
                 IndicesModule.getNamedWriteables().stream(),
                 searchModule.getNamedWriteables().stream(),
                 pluginsService.flatMap(Plugin::getNamedWriteables),
-                ClusterModule.getNamedWriteables().stream(),
-                SystemIndexMigrationExecutor.getNamedWriteables().stream()
+                ClusterModule.getNamedWriteables().stream()
             ).flatMap(Function.identity()).toList()
         );
         xContentRegistry = new NamedXContentRegistry(
@@ -588,7 +586,6 @@ private void createClientAndRegistries(Settings settings, ThreadPool threadPool,
                 searchModule.getNamedXContents().stream(),
                 pluginsService.flatMap(Plugin::getNamedXContent),
                 ClusterModule.getNamedXWriteables().stream(),
-                SystemIndexMigrationExecutor.getNamedXContentParsers().stream(),
                 HealthNodeTaskExecutor.getNamedXContentParsers().stream()
             ).flatMap(Function.identity()).toList()
         );
@@ -1183,17 +1180,7 @@ public Map<String, String> queryFields() {
 
         final ShutdownPrepareService shutdownPrepareService = new ShutdownPrepareService(settings, httpServerTransport, terminationHandler);
 
-        modules.add(
-            loadPersistentTasksService(
-                settingsModule,
-                clusterService,
-                threadPool,
-                systemIndices,
-                clusterModule.getIndexNameExpressionResolver(),
-                metadataUpdateSettingsService,
-                metadataCreateIndexService
-            )
-        );
+        modules.add(loadPersistentTasksService(settingsModule, clusterService, threadPool, clusterModule.getIndexNameExpressionResolver()));
 
         modules.add(
             loadPluginShutdownService(clusterService),
@@ -1696,27 +1683,16 @@ private Module loadPersistentTasksService(
         SettingsModule settingsModule,
         ClusterService clusterService,
         ThreadPool threadPool,
-        SystemIndices systemIndices,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        MetadataUpdateSettingsService metadataUpdateSettingsService,
-        MetadataCreateIndexService metadataCreateIndexService
+        IndexNameExpressionResolver indexNameExpressionResolver
     ) {
         PersistentTasksService persistentTasksService = new PersistentTasksService(clusterService, threadPool, client);
-        SystemIndexMigrationExecutor systemIndexMigrationExecutor = new SystemIndexMigrationExecutor(
-            client,
-            clusterService,
-            systemIndices,
-            metadataUpdateSettingsService,
-            metadataCreateIndexService,
-            settingsModule.getIndexScopedSettings()
-        );
         HealthNodeTaskExecutor healthNodeTaskExecutor = HealthNodeTaskExecutor.create(
             clusterService,
             persistentTasksService,
             settingsModule.getSettings(),
             clusterService.getClusterSettings()
         );
-        Stream<PersistentTasksExecutor<?>> builtinTaskExecutors = Stream.of(systemIndexMigrationExecutor, healthNodeTaskExecutor);
+        Stream<PersistentTasksExecutor<?>> builtinTaskExecutors = Stream.of(healthNodeTaskExecutor);
 
         Stream<PersistentTasksExecutor<?>> pluginTaskExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class)
             .map(p -> p.getPersistentTasksExecutor(clusterService, threadPool, client, settingsModule, indexNameExpressionResolver))