Examples with ClusterStateUpdateTask org.opensearch.cluster.ClusterStateUpdateTask used on opensource projects

Example 1 with ClusterStateUpdateTask

use of org.opensearch.cluster.ClusterStateUpdateTask in project OpenSearch by opensearch-project.

the class ClusterServiceIT method testPendingUpdateTask.

public void testPendingUpdateTask() throws Exception {
    String node_0 = internalCluster().startNode();
    internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
    final ClusterService clusterService = internalCluster().getInstance(ClusterService.class, node_0);
    final CountDownLatch block1 = new CountDownLatch(1);
    final CountDownLatch invoked1 = new CountDownLatch(1);
    clusterService.submitStateUpdateTask("1", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) {
            invoked1.countDown();
            try {
                block1.await();
            } catch (InterruptedException e) {
                fail();
            }
            return currentState;
        }

        @Override
        public void onFailure(String source, Exception e) {
            invoked1.countDown();
            fail();
        }
    });
    invoked1.await();
    final CountDownLatch invoked2 = new CountDownLatch(9);
    for (int i = 2; i <= 10; i++) {
        clusterService.submitStateUpdateTask(Integer.toString(i), new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                return currentState;
            }

            @Override
            public void onFailure(String source, Exception e) {
                fail();
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                invoked2.countDown();
            }
        });
    }
    // there might be other tasks in this node, make sure to only take the ones we add into account in this test
    // The tasks can be re-ordered, so we need to check out-of-order
    Set<String> controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
    List<PendingClusterTask> pendingClusterTasks = clusterService.getMasterService().pendingTasks();
    assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(10));
    assertThat(pendingClusterTasks.get(0).getSource().string(), equalTo("1"));
    assertThat(pendingClusterTasks.get(0).isExecuting(), equalTo(true));
    for (PendingClusterTask task : pendingClusterTasks) {
        controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());
    controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
    PendingClusterTasksResponse response = internalCluster().coordOnlyNodeClient().admin().cluster().preparePendingClusterTasks().get();
    assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(10));
    assertThat(response.pendingTasks().get(0).getSource().string(), equalTo("1"));
    assertThat(response.pendingTasks().get(0).isExecuting(), equalTo(true));
    for (PendingClusterTask task : response) {
        controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());
    block1.countDown();
    invoked2.await();
    // whenever we test for no tasks, we need to wait since this is a live node
    assertBusy(() -> assertTrue("Pending tasks not empty", clusterService.getMasterService().pendingTasks().isEmpty()));
    waitNoPendingTasksOnAll();
    final CountDownLatch block2 = new CountDownLatch(1);
    final CountDownLatch invoked3 = new CountDownLatch(1);
    clusterService.submitStateUpdateTask("1", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) {
            invoked3.countDown();
            try {
                block2.await();
            } catch (InterruptedException e) {
                fail();
            }
            return currentState;
        }

        @Override
        public void onFailure(String source, Exception e) {
            invoked3.countDown();
            fail();
        }
    });
    invoked3.await();
    for (int i = 2; i <= 5; i++) {
        clusterService.submitStateUpdateTask(Integer.toString(i), new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                return currentState;
            }

            @Override
            public void onFailure(String source, Exception e) {
                fail();
            }
        });
    }
    Thread.sleep(100);
    pendingClusterTasks = clusterService.getMasterService().pendingTasks();
    assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(5));
    controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
    for (PendingClusterTask task : pendingClusterTasks) {
        controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());
    response = internalCluster().coordOnlyNodeClient().admin().cluster().preparePendingClusterTasks().get();
    assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(5));
    controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
    for (PendingClusterTask task : response) {
        if (controlSources.remove(task.getSource().string())) {
            assertThat(task.getTimeInQueueInMillis(), greaterThan(0L));
        }
    }
    assertTrue(controlSources.isEmpty());
    block2.countDown();
}

Example 2 with ClusterStateUpdateTask

use of org.opensearch.cluster.ClusterStateUpdateTask in project OpenSearch by opensearch-project.

the class ShardStateActionIT method testFollowupRerouteCanBeSetToHigherPriority.

public void testFollowupRerouteCanBeSetToHigherPriority() {
    // Shows that in a cluster under unbearable pressure we can still assign replicas (for now at least) by setting
    // cluster.routing.allocation.shard_state.reroute.priority to a higher priority. Can be removed when this setting is removed, as
    // we should at that point be confident that the default priority is appropriate for all clusters.
    internalCluster().ensureAtLeastNumDataNodes(2);
    assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().put(ShardStateAction.FOLLOW_UP_REROUTE_PRIORITY_SETTING.getKey(), "urgent")));
    // ensure that the master always has a HIGH priority pending task
    final AtomicBoolean stopSpammingMaster = new AtomicBoolean();
    final ClusterService masterClusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName());
    masterClusterService.submitStateUpdateTask("spam", new ClusterStateUpdateTask(Priority.HIGH) {

        @Override
        public ClusterState execute(ClusterState currentState) {
            return currentState;
        }

        @Override
        public void onFailure(String source, Exception e) {
            throw new AssertionError(source, e);
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            if (stopSpammingMaster.get() == false) {
                masterClusterService.submitStateUpdateTask("spam", this);
            }
        }
    });
    // even with the master under such pressure, all shards of the index can be assigned; in particular, after the primaries have
    // started there's a follow-up reroute at a higher priority than the spam
    createIndex("test");
    assertFalse(client().admin().cluster().prepareHealth().setWaitForGreenStatus().get().isTimedOut());
    stopSpammingMaster.set(true);
    assertFalse(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get().isTimedOut());
    assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().putNull(ShardStateAction.FOLLOW_UP_REROUTE_PRIORITY_SETTING.getKey())));
}

Example 3 with ClusterStateUpdateTask

use of org.opensearch.cluster.ClusterStateUpdateTask in project OpenSearch by opensearch-project.

the class TransportCleanupRepositoryAction method cleanupRepo.

/**
 * Runs cleanup operations on the given repository.
 * @param repositoryName Repository to clean up
 * @param listener Listener for cleanup result
 */
private void cleanupRepo(String repositoryName, ActionListener<RepositoryCleanupResult> listener) {
    final Repository repository = repositoriesService.repository(repositoryName);
    if (repository instanceof BlobStoreRepository == false) {
        listener.onFailure(new IllegalArgumentException("Repository [" + repositoryName + "] does not support repository cleanup"));
        return;
    }
    final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository;
    final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
    repository.getRepositoryData(repositoryDataListener);
    repositoryDataListener.whenComplete(repositoryData -> {
        final long repositoryStateId = repositoryData.getGenId();
        logger.info("Running cleanup operations on repository [{}][{}]", repositoryName, repositoryStateId);
        clusterService.submitStateUpdateTask("cleanup repository [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {

            private boolean startedCleanup = false;

            @Override
            public ClusterState execute(ClusterState currentState) {
                final RepositoryCleanupInProgress repositoryCleanupInProgress = currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY);
                if (repositoryCleanupInProgress.hasCleanupInProgress()) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a repository cleanup is already in-progress in [" + repositoryCleanupInProgress + "]");
                }
                final SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY);
                if (deletionsInProgress.hasDeletionsInProgress()) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently being deleted in [" + deletionsInProgress + "]");
                }
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
                if (snapshots.entries().isEmpty() == false) {
                    throw new IllegalStateException("Cannot cleanup [" + repositoryName + "] - a snapshot is currently running in [" + snapshots + "]");
                }
                return ClusterState.builder(currentState).putCustom(RepositoryCleanupInProgress.TYPE, new RepositoryCleanupInProgress(Collections.singletonList(RepositoryCleanupInProgress.startedEntry(repositoryName, repositoryStateId)))).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                after(e, null);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                startedCleanup = true;
                logger.debug("Initialized repository cleanup in cluster state for [{}][{}]", repositoryName, repositoryStateId);
                threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> blobStoreRepository.cleanup(repositoryStateId, snapshotsService.minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null), ActionListener.wrap(result -> after(null, result), e -> after(e, null)))));
            }

            private void after(@Nullable Exception failure, @Nullable RepositoryCleanupResult result) {
                if (failure == null) {
                    logger.debug("Finished repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId);
                } else {
                    logger.debug(() -> new ParameterizedMessage("Failed to finish repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
                }
                assert failure != null || result != null;
                if (startedCleanup == false) {
                    logger.debug("No cleanup task to remove from cluster state because we failed to start one", failure);
                    listener.onFailure(failure);
                    return;
                }
                clusterService.submitStateUpdateTask("remove repository cleanup task [" + repositoryName + "][" + repositoryStateId + ']', new ClusterStateUpdateTask() {

                    @Override
                    public ClusterState execute(ClusterState currentState) {
                        return removeInProgressCleanup(currentState);
                    }

                    @Override
                    public void onFailure(String source, Exception e) {
                        if (failure != null) {
                            e.addSuppressed(failure);
                        }
                        logger.warn(() -> new ParameterizedMessage("[{}] failed to remove repository cleanup task", repositoryName), e);
                        listener.onFailure(e);
                    }

                    @Override
                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                        if (failure == null) {
                            logger.info("Done with repository cleanup on [{}][{}] with result [{}]", repositoryName, repositoryStateId, result);
                            listener.onResponse(result);
                        } else {
                            logger.warn(() -> new ParameterizedMessage("Failed to run repository cleanup operations on [{}][{}]", repositoryName, repositoryStateId), failure);
                            listener.onFailure(failure);
                        }
                    }
                });
            }
        });
    }, listener::onFailure);
}

Example 4 with ClusterStateUpdateTask

use of org.opensearch.cluster.ClusterStateUpdateTask in project OpenSearch by opensearch-project.

the class TransportRolloverAction method masterOperation.

@Override
protected void masterOperation(Task task, final RolloverRequest rolloverRequest, final ClusterState state, final ActionListener<RolloverResponse> listener) throws Exception {
    MetadataRolloverService.RolloverResult preResult = rolloverService.rolloverClusterState(state, rolloverRequest.getRolloverTarget(), rolloverRequest.getNewIndexName(), rolloverRequest.getCreateIndexRequest(), Collections.emptyList(), true, true);
    Metadata metadata = state.metadata();
    String sourceIndexName = preResult.sourceIndexName;
    String rolloverIndexName = preResult.rolloverIndexName;
    IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getRolloverTarget()).clear().indicesOptions(IndicesOptions.fromOptions(true, false, true, true)).docs(true);
    statsRequest.setParentTask(clusterService.localNode().getId(), task.getId());
    client.execute(IndicesStatsAction.INSTANCE, statsRequest, new ActionListener<IndicesStatsResponse>() {

        @Override
        public void onResponse(IndicesStatsResponse statsResponse) {
            final Map<String, Boolean> conditionResults = evaluateConditions(rolloverRequest.getConditions().values(), metadata.index(sourceIndexName), statsResponse);
            if (rolloverRequest.isDryRun()) {
                listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, true, false, false, false));
                return;
            }
            List<Condition<?>> metConditions = rolloverRequest.getConditions().values().stream().filter(condition -> conditionResults.get(condition.toString())).collect(Collectors.toList());
            if (conditionResults.size() == 0 || metConditions.size() > 0) {
                clusterService.submitStateUpdateTask("rollover_index source [" + sourceIndexName + "] to target [" + rolloverIndexName + "]", new ClusterStateUpdateTask() {

                    @Override
                    public ClusterState execute(ClusterState currentState) throws Exception {
                        MetadataRolloverService.RolloverResult rolloverResult = rolloverService.rolloverClusterState(currentState, rolloverRequest.getRolloverTarget(), rolloverRequest.getNewIndexName(), rolloverRequest.getCreateIndexRequest(), metConditions, false, false);
                        if (rolloverResult.sourceIndexName.equals(sourceIndexName) == false) {
                            throw new OpenSearchException("Concurrent modification of alias [{}] during rollover", rolloverRequest.getRolloverTarget());
                        }
                        return rolloverResult.clusterState;
                    }

                    @Override
                    public void onFailure(String source, Exception e) {
                        listener.onFailure(e);
                    }

                    @Override
                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                        if (newState.equals(oldState) == false) {
                            activeShardsObserver.waitForActiveShards(new String[] { rolloverIndexName }, rolloverRequest.getCreateIndexRequest().waitForActiveShards(), rolloverRequest.masterNodeTimeout(), isShardsAcknowledged -> listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, true, true, isShardsAcknowledged)), listener::onFailure);
                        }
                    }
                });
            } else {
                // conditions not met
                listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, false, false, false));
            }
        }

        @Override
        public void onFailure(Exception e) {
            listener.onFailure(e);
        }
    });
}

Example 5 with ClusterStateUpdateTask

use of org.opensearch.cluster.ClusterStateUpdateTask in project OpenSearch by opensearch-project.

the class MetadataIndexStateService method closeIndices.

/**
 * Closes one or more indices.
 *
 * Closing indices is a 3 steps process: it first adds a write block to every indices to close, then waits for the operations on shards
 * to be terminated and finally closes the indices by moving their state to CLOSE.
 */
public void closeIndices(final CloseIndexClusterStateUpdateRequest request, final ActionListener<CloseIndexResponse> listener) {
    final Index[] concreteIndices = request.indices();
    if (concreteIndices == null || concreteIndices.length == 0) {
        throw new IllegalArgumentException("Index name is required");
    }
    List<String> writeIndices = new ArrayList<>();
    SortedMap<String, IndexAbstraction> lookup = clusterService.state().metadata().getIndicesLookup();
    for (Index index : concreteIndices) {
        IndexAbstraction ia = lookup.get(index.getName());
        if (ia != null && ia.getParentDataStream() != null && ia.getParentDataStream().getWriteIndex().getIndex().equals(index)) {
            writeIndices.add(index.getName());
        }
    }
    if (writeIndices.size() > 0) {
        throw new IllegalArgumentException("cannot close the following data stream write indices [" + Strings.collectionToCommaDelimitedString(writeIndices) + "]");
    }
    clusterService.submitStateUpdateTask("add-block-index-to-close " + Arrays.toString(concreteIndices), new ClusterStateUpdateTask(Priority.URGENT) {

        private final Map<Index, ClusterBlock> blockedIndices = new HashMap<>();

        @Override
        public ClusterState execute(final ClusterState currentState) {
            return addIndexClosedBlocks(concreteIndices, blockedIndices, currentState);
        }

        @Override
        public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) {
            if (oldState == newState) {
                assert blockedIndices.isEmpty() : "List of blocked indices is not empty but cluster state wasn't changed";
                listener.onResponse(new CloseIndexResponse(true, false, Collections.emptyList()));
            } else {
                assert blockedIndices.isEmpty() == false : "List of blocked indices is empty but cluster state was changed";
                threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new WaitForClosedBlocksApplied(blockedIndices, request, ActionListener.wrap(verifyResults -> clusterService.submitStateUpdateTask("close-indices", new ClusterStateUpdateTask(Priority.URGENT) {

                    private final List<IndexResult> indices = new ArrayList<>();

                    @Override
                    public ClusterState execute(final ClusterState currentState) throws Exception {
                        Tuple<ClusterState, Collection<IndexResult>> closingResult = closeRoutingTable(currentState, blockedIndices, verifyResults);
                        assert verifyResults.size() == closingResult.v2().size();
                        indices.addAll(closingResult.v2());
                        return allocationService.reroute(closingResult.v1(), "indices closed");
                    }

                    @Override
                    public void onFailure(final String source, final Exception e) {
                        listener.onFailure(e);
                    }

                    @Override
                    public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) {
                        final boolean acknowledged = indices.stream().noneMatch(IndexResult::hasFailures);
                        final String[] waitForIndices = indices.stream().filter(result -> result.hasFailures() == false).filter(result -> newState.routingTable().hasIndex(result.getIndex())).map(result -> result.getIndex().getName()).toArray(String[]::new);
                        if (waitForIndices.length > 0) {
                            activeShardsObserver.waitForActiveShards(waitForIndices, request.waitForActiveShards(), request.ackTimeout(), shardsAcknowledged -> {
                                if (shardsAcknowledged == false) {
                                    logger.debug("[{}] indices closed, but the operation timed out while waiting " + "for enough shards to be started.", Arrays.toString(waitForIndices));
                                }
                                // acknowledged maybe be false but some indices may have been correctly
                                // closed, so
                                // we maintain a kind of coherency by overriding the shardsAcknowledged
                                // value
                                // (see ShardsAcknowledgedResponse constructor)
                                boolean shardsAcked = acknowledged ? shardsAcknowledged : false;
                                listener.onResponse(new CloseIndexResponse(acknowledged, shardsAcked, indices));
                            }, listener::onFailure);
                        } else {
                            listener.onResponse(new CloseIndexResponse(acknowledged, false, indices));
                        }
                    }
                }), listener::onFailure)));
            }
        }

        @Override
        public void onFailure(final String source, final Exception e) {
            listener.onFailure(e);
        }

        @Override
        public TimeValue timeout() {
            return request.masterNodeTimeout();
        }
    });
}