Examples with ClusterStateUpdateTask org.elasticsearch.cluster.ClusterStateUpdateTask used on opensource projects

Example 6 with ClusterStateUpdateTask

use of org.elasticsearch.cluster.ClusterStateUpdateTask in project elasticsearch by elastic.

the class SnapshotsService method beginSnapshot.

/**
     * Starts snapshot.
     * <p>
     * Creates snapshot in repository and updates snapshot metadata record with list of shards that needs to be processed.
     *
     * @param clusterState               cluster state
     * @param snapshot                   snapshot meta data
     * @param partial                    allow partial snapshots
     * @param userCreateSnapshotListener listener
     */
private void beginSnapshot(final ClusterState clusterState, final SnapshotsInProgress.Entry snapshot, final boolean partial, final CreateSnapshotListener userCreateSnapshotListener) {
    boolean snapshotCreated = false;
    try {
        Repository repository = repositoriesService.repository(snapshot.snapshot().getRepository());
        MetaData metaData = clusterState.metaData();
        if (!snapshot.includeGlobalState()) {
            // Remove global state from the cluster state
            MetaData.Builder builder = MetaData.builder();
            for (IndexId index : snapshot.indices()) {
                builder.put(metaData.index(index.getName()), false);
            }
            metaData = builder.build();
        }
        repository.initializeSnapshot(snapshot.snapshot().getSnapshotId(), snapshot.indices(), metaData);
        snapshotCreated = true;
        if (snapshot.indices().isEmpty()) {
            // No indices in this snapshot - we are done
            userCreateSnapshotListener.onResponse();
            endSnapshot(snapshot);
            return;
        }
        clusterService.submitStateUpdateTask("update_snapshot [" + snapshot.snapshot() + "]", new ClusterStateUpdateTask() {

            boolean accepted = false;

            SnapshotsInProgress.Entry updatedSnapshot;

            String failure = null;

            @Override
            public ClusterState execute(ClusterState currentState) {
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
                    if (entry.snapshot().equals(snapshot.snapshot())) {
                        // Replace the snapshot that was just created
                        ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = shards(currentState, entry.indices());
                        if (!partial) {
                            Tuple<Set<String>, Set<String>> indicesWithMissingShards = indicesWithMissingShards(shards, currentState.metaData());
                            Set<String> missing = indicesWithMissingShards.v1();
                            Set<String> closed = indicesWithMissingShards.v2();
                            if (missing.isEmpty() == false || closed.isEmpty() == false) {
                                StringBuilder failureMessage = new StringBuilder();
                                updatedSnapshot = new SnapshotsInProgress.Entry(entry, State.FAILED, shards);
                                entries.add(updatedSnapshot);
                                if (missing.isEmpty() == false) {
                                    failureMessage.append("Indices don't have primary shards ");
                                    failureMessage.append(missing);
                                }
                                if (closed.isEmpty() == false) {
                                    if (failureMessage.length() > 0) {
                                        failureMessage.append("; ");
                                    }
                                    failureMessage.append("Indices are closed ");
                                    failureMessage.append(closed);
                                }
                                failure = failureMessage.toString();
                                continue;
                            }
                        }
                        updatedSnapshot = new SnapshotsInProgress.Entry(entry, State.STARTED, shards);
                        entries.add(updatedSnapshot);
                        if (!completed(shards.values())) {
                            accepted = true;
                        }
                    } else {
                        entries.add(entry);
                    }
                }
                return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to create snapshot", snapshot.snapshot().getSnapshotId()), e);
                removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, e));
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                // The userCreateSnapshotListener.onResponse() notifies caller that the snapshot was accepted
                // for processing. If client wants to wait for the snapshot completion, it can register snapshot
                // completion listener in this method. For the snapshot completion to work properly, the snapshot
                // should still exist when listener is registered.
                userCreateSnapshotListener.onResponse();
                // go ahead and continue working on this snapshot rather then end here.
                if (!accepted && updatedSnapshot != null) {
                    endSnapshot(updatedSnapshot, failure);
                }
            }
        });
    } catch (Exception e) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to create snapshot [{}]", snapshot.snapshot().getSnapshotId()), e);
        removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, e));
    }
}

Example 7 with ClusterStateUpdateTask

use of org.elasticsearch.cluster.ClusterStateUpdateTask in project elasticsearch by elastic.

the class MetaDataIndexTemplateService method putTemplate.

public void putTemplate(final PutRequest request, final PutListener listener) {
    Settings.Builder updatedSettingsBuilder = Settings.builder();
    updatedSettingsBuilder.put(request.settings).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
    request.settings(updatedSettingsBuilder.build());
    if (request.name == null) {
        listener.onFailure(new IllegalArgumentException("index_template must provide a name"));
        return;
    }
    if (request.indexPatterns == null) {
        listener.onFailure(new IllegalArgumentException("index_template must provide a template"));
        return;
    }
    try {
        validate(request);
    } catch (Exception e) {
        listener.onFailure(e);
        return;
    }
    final IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(request.name);
    clusterService.submitStateUpdateTask("create-index-template [" + request.name + "], cause [" + request.cause + "]", new ClusterStateUpdateTask(Priority.URGENT) {

        @Override
        public TimeValue timeout() {
            return request.masterTimeout;
        }

        @Override
        public void onFailure(String source, Exception e) {
            listener.onFailure(e);
        }

        @Override
        public ClusterState execute(ClusterState currentState) throws Exception {
            if (request.create && currentState.metaData().templates().containsKey(request.name)) {
                throw new IllegalArgumentException("index_template [" + request.name + "] already exists");
            }
            validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
            for (Alias alias : request.aliases) {
                AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter()).indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
                templateBuilder.putAlias(aliasMetaData);
            }
            for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
                templateBuilder.putCustom(entry.getKey(), entry.getValue());
            }
            IndexTemplateMetaData template = templateBuilder.build();
            MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template);
            return ClusterState.builder(currentState).metaData(builder).build();
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            listener.onResponse(new PutResponse(true, templateBuilder.build()));
        }
    });
}

Example 8 with ClusterStateUpdateTask

use of org.elasticsearch.cluster.ClusterStateUpdateTask in project elasticsearch by elastic.

the class ClusterServiceIT method testPendingUpdateTask.

@TestLogging("_root:debug,org.elasticsearch.action.admin.cluster.tasks:trace")
public void testPendingUpdateTask() throws Exception {
    String node_0 = internalCluster().startNode();
    internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
    final ClusterService clusterService = internalCluster().getInstance(ClusterService.class, node_0);
    final CountDownLatch block1 = new CountDownLatch(1);
    final CountDownLatch invoked1 = new CountDownLatch(1);
    clusterService.submitStateUpdateTask("1", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) {
            invoked1.countDown();
            try {
                block1.await();
            } catch (InterruptedException e) {
                fail();
            }
            return currentState;
        }

        @Override
        public void onFailure(String source, Exception e) {
            invoked1.countDown();
            fail();
        }
    });
    invoked1.await();
    final CountDownLatch invoked2 = new CountDownLatch(9);
    for (int i = 2; i <= 10; i++) {
        clusterService.submitStateUpdateTask(Integer.toString(i), new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                return currentState;
            }

            @Override
            public void onFailure(String source, Exception e) {
                fail();
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                invoked2.countDown();
            }
        });
    }
    // there might be other tasks in this node, make sure to only take the ones we add into account in this test
    // The tasks can be re-ordered, so we need to check out-of-order
    Set<String> controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
    List<PendingClusterTask> pendingClusterTasks = clusterService.pendingTasks();
    assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(10));
    assertThat(pendingClusterTasks.get(0).getSource().string(), equalTo("1"));
    assertThat(pendingClusterTasks.get(0).isExecuting(), equalTo(true));
    for (PendingClusterTask task : pendingClusterTasks) {
        controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());
    controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
    PendingClusterTasksResponse response = internalCluster().coordOnlyNodeClient().admin().cluster().preparePendingClusterTasks().get();
    assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(10));
    assertThat(response.pendingTasks().get(0).getSource().string(), equalTo("1"));
    assertThat(response.pendingTasks().get(0).isExecuting(), equalTo(true));
    for (PendingClusterTask task : response) {
        controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());
    block1.countDown();
    invoked2.await();
    // whenever we test for no tasks, we need to awaitBusy since this is a live node
    assertTrue(awaitBusy(() -> clusterService.pendingTasks().isEmpty()));
    waitNoPendingTasksOnAll();
    final CountDownLatch block2 = new CountDownLatch(1);
    final CountDownLatch invoked3 = new CountDownLatch(1);
    clusterService.submitStateUpdateTask("1", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) {
            invoked3.countDown();
            try {
                block2.await();
            } catch (InterruptedException e) {
                fail();
            }
            return currentState;
        }

        @Override
        public void onFailure(String source, Exception e) {
            invoked3.countDown();
            fail();
        }
    });
    invoked3.await();
    for (int i = 2; i <= 5; i++) {
        clusterService.submitStateUpdateTask(Integer.toString(i), new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) {
                return currentState;
            }

            @Override
            public void onFailure(String source, Exception e) {
                fail();
            }
        });
    }
    Thread.sleep(100);
    pendingClusterTasks = clusterService.pendingTasks();
    assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(5));
    controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
    for (PendingClusterTask task : pendingClusterTasks) {
        controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());
    response = internalCluster().coordOnlyNodeClient().admin().cluster().preparePendingClusterTasks().get();
    assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(5));
    controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
    for (PendingClusterTask task : response) {
        if (controlSources.remove(task.getSource().string())) {
            assertThat(task.getTimeInQueueInMillis(), greaterThan(0L));
        }
    }
    assertTrue(controlSources.isEmpty());
    block2.countDown();
}

Example 9 with ClusterStateUpdateTask

use of org.elasticsearch.cluster.ClusterStateUpdateTask in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testDeleteOrphanSnapshot.

public void testDeleteOrphanSnapshot() throws Exception {
    Client client = client();
    logger.info("-->  creating repository");
    final String repositoryName = "test-repo";
    assertAcked(client.admin().cluster().preparePutRepository(repositoryName).setType("mock").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
    logger.info("--> create the index");
    final String idxName = "test-idx";
    createIndex(idxName);
    ensureGreen();
    ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName());
    final CountDownLatch countDownLatch = new CountDownLatch(1);
    logger.info("--> snapshot");
    final String snapshotName = "test-snap";
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName).setWaitForCompletion(true).setIndices(idxName).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    logger.info("--> emulate an orphan snapshot");
    RepositoriesService repositoriesService = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    final RepositoryData repositoryData = repositoriesService.repository(repositoryName).getRepositoryData();
    final IndexId indexId = repositoryData.resolveIndexId(idxName);
    clusterService.submitStateUpdateTask("orphan snapshot test", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) {
            // Simulate orphan snapshot
            ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
            shards.put(new ShardId(idxName, "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            shards.put(new ShardId(idxName, "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            shards.put(new ShardId(idxName, "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            List<Entry> entries = new ArrayList<>();
            entries.add(new Entry(new Snapshot(repositoryName, createSnapshotResponse.getSnapshotInfo().snapshotId()), true, false, State.ABORTED, Collections.singletonList(indexId), System.currentTimeMillis(), repositoryData.getGenId(), shards.build()));
            return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            fail();
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) {
            countDownLatch.countDown();
        }
    });
    countDownLatch.await();
    logger.info("--> try deleting the orphan snapshot");
    assertAcked(client.admin().cluster().prepareDeleteSnapshot(repositoryName, snapshotName).get("10s"));
}

Example 10 with ClusterStateUpdateTask

use of org.elasticsearch.cluster.ClusterStateUpdateTask in project elasticsearch by elastic.

the class ClusterServiceTests method testClusterStateApplierCantSampleClusterState.

public void testClusterStateApplierCantSampleClusterState() throws InterruptedException {
    AtomicReference<Throwable> error = new AtomicReference<>();
    AtomicBoolean applierCalled = new AtomicBoolean();
    clusterService.addStateApplier(event -> {
        try {
            applierCalled.set(true);
            clusterService.state();
            error.set(new AssertionError("successfully sampled state"));
        } catch (AssertionError e) {
            if (e.getMessage().contains("should not be called by a cluster state applier") == false) {
                error.set(e);
            }
        }
    });
    CountDownLatch latch = new CountDownLatch(1);
    clusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) throws Exception {
            return ClusterState.builder(currentState).build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            error.compareAndSet(null, e);
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            latch.countDown();
        }
    });
    latch.await();
    assertNull(error.get());
    assertTrue(applierCalled.get());
}