Examples with SnapshotsInProgress org.elasticsearch.cluster.SnapshotsInProgress used on opensource projects

Example 6 with SnapshotsInProgress

use of org.elasticsearch.cluster.SnapshotsInProgress in project elasticsearch by elastic.

the class SnapshotsService method beginSnapshot.

/**
     * Starts snapshot.
     * <p>
     * Creates snapshot in repository and updates snapshot metadata record with list of shards that needs to be processed.
     *
     * @param clusterState               cluster state
     * @param snapshot                   snapshot meta data
     * @param partial                    allow partial snapshots
     * @param userCreateSnapshotListener listener
     */
private void beginSnapshot(final ClusterState clusterState, final SnapshotsInProgress.Entry snapshot, final boolean partial, final CreateSnapshotListener userCreateSnapshotListener) {
    boolean snapshotCreated = false;
    try {
        Repository repository = repositoriesService.repository(snapshot.snapshot().getRepository());
        MetaData metaData = clusterState.metaData();
        if (!snapshot.includeGlobalState()) {
            // Remove global state from the cluster state
            MetaData.Builder builder = MetaData.builder();
            for (IndexId index : snapshot.indices()) {
                builder.put(metaData.index(index.getName()), false);
            }
            metaData = builder.build();
        }
        repository.initializeSnapshot(snapshot.snapshot().getSnapshotId(), snapshot.indices(), metaData);
        snapshotCreated = true;
        if (snapshot.indices().isEmpty()) {
            // No indices in this snapshot - we are done
            userCreateSnapshotListener.onResponse();
            endSnapshot(snapshot);
            return;
        }
        clusterService.submitStateUpdateTask("update_snapshot [" + snapshot.snapshot() + "]", new ClusterStateUpdateTask() {

            boolean accepted = false;

            SnapshotsInProgress.Entry updatedSnapshot;

            String failure = null;

            @Override
            public ClusterState execute(ClusterState currentState) {
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
                    if (entry.snapshot().equals(snapshot.snapshot())) {
                        // Replace the snapshot that was just created
                        ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = shards(currentState, entry.indices());
                        if (!partial) {
                            Tuple<Set<String>, Set<String>> indicesWithMissingShards = indicesWithMissingShards(shards, currentState.metaData());
                            Set<String> missing = indicesWithMissingShards.v1();
                            Set<String> closed = indicesWithMissingShards.v2();
                            if (missing.isEmpty() == false || closed.isEmpty() == false) {
                                StringBuilder failureMessage = new StringBuilder();
                                updatedSnapshot = new SnapshotsInProgress.Entry(entry, State.FAILED, shards);
                                entries.add(updatedSnapshot);
                                if (missing.isEmpty() == false) {
                                    failureMessage.append("Indices don't have primary shards ");
                                    failureMessage.append(missing);
                                }
                                if (closed.isEmpty() == false) {
                                    if (failureMessage.length() > 0) {
                                        failureMessage.append("; ");
                                    }
                                    failureMessage.append("Indices are closed ");
                                    failureMessage.append(closed);
                                }
                                failure = failureMessage.toString();
                                continue;
                            }
                        }
                        updatedSnapshot = new SnapshotsInProgress.Entry(entry, State.STARTED, shards);
                        entries.add(updatedSnapshot);
                        if (!completed(shards.values())) {
                            accepted = true;
                        }
                    } else {
                        entries.add(entry);
                    }
                }
                return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to create snapshot", snapshot.snapshot().getSnapshotId()), e);
                removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, e));
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                // The userCreateSnapshotListener.onResponse() notifies caller that the snapshot was accepted
                // for processing. If client wants to wait for the snapshot completion, it can register snapshot
                // completion listener in this method. For the snapshot completion to work properly, the snapshot
                // should still exist when listener is registered.
                userCreateSnapshotListener.onResponse();
                // go ahead and continue working on this snapshot rather then end here.
                if (!accepted && updatedSnapshot != null) {
                    endSnapshot(updatedSnapshot, failure);
                }
            }
        });
    } catch (Exception e) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to create snapshot [{}]", snapshot.snapshot().getSnapshotId()), e);
        removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, e));
    }
}

Example 7 with SnapshotsInProgress

use of org.elasticsearch.cluster.SnapshotsInProgress in project elasticsearch by elastic.

the class SnapshotsService method currentSnapshots.

/**
     * Returns status of the currently running snapshots
     * <p>
     * This method is executed on master node
     * </p>
     *
     * @param repository repository id
     * @param snapshots  list of snapshots that will be used as a filter, empty list means no snapshots are filtered
     * @return list of metadata for currently running snapshots
     */
public List<SnapshotsInProgress.Entry> currentSnapshots(final String repository, final List<String> snapshots) {
    SnapshotsInProgress snapshotsInProgress = clusterService.state().custom(SnapshotsInProgress.TYPE);
    if (snapshotsInProgress == null || snapshotsInProgress.entries().isEmpty()) {
        return Collections.emptyList();
    }
    if ("_all".equals(repository)) {
        return snapshotsInProgress.entries();
    }
    if (snapshotsInProgress.entries().size() == 1) {
        // Most likely scenario - one snapshot is currently running
        // Check this snapshot against the query
        SnapshotsInProgress.Entry entry = snapshotsInProgress.entries().get(0);
        if (entry.snapshot().getRepository().equals(repository) == false) {
            return Collections.emptyList();
        }
        if (snapshots.isEmpty() == false) {
            for (String snapshot : snapshots) {
                if (entry.snapshot().getSnapshotId().getName().equals(snapshot)) {
                    return snapshotsInProgress.entries();
                }
            }
            return Collections.emptyList();
        } else {
            return snapshotsInProgress.entries();
        }
    }
    List<SnapshotsInProgress.Entry> builder = new ArrayList<>();
    for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
        if (entry.snapshot().getRepository().equals(repository) == false) {
            continue;
        }
        if (snapshots.isEmpty() == false) {
            for (String snapshot : snapshots) {
                if (entry.snapshot().getSnapshotId().getName().equals(snapshot)) {
                    builder.add(entry);
                    break;
                }
            }
        } else {
            builder.add(entry);
        }
    }
    return Collections.unmodifiableList(builder);
}

Example 8 with SnapshotsInProgress

use of org.elasticsearch.cluster.SnapshotsInProgress in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testDeleteOrphanSnapshot.

public void testDeleteOrphanSnapshot() throws Exception {
    Client client = client();
    logger.info("-->  creating repository");
    final String repositoryName = "test-repo";
    assertAcked(client.admin().cluster().preparePutRepository(repositoryName).setType("mock").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
    logger.info("--> create the index");
    final String idxName = "test-idx";
    createIndex(idxName);
    ensureGreen();
    ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName());
    final CountDownLatch countDownLatch = new CountDownLatch(1);
    logger.info("--> snapshot");
    final String snapshotName = "test-snap";
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName).setWaitForCompletion(true).setIndices(idxName).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    logger.info("--> emulate an orphan snapshot");
    RepositoriesService repositoriesService = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    final RepositoryData repositoryData = repositoriesService.repository(repositoryName).getRepositoryData();
    final IndexId indexId = repositoryData.resolveIndexId(idxName);
    clusterService.submitStateUpdateTask("orphan snapshot test", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) {
            // Simulate orphan snapshot
            ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
            shards.put(new ShardId(idxName, "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            shards.put(new ShardId(idxName, "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            shards.put(new ShardId(idxName, "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            List<Entry> entries = new ArrayList<>();
            entries.add(new Entry(new Snapshot(repositoryName, createSnapshotResponse.getSnapshotInfo().snapshotId()), true, false, State.ABORTED, Collections.singletonList(indexId), System.currentTimeMillis(), repositoryData.getGenId(), shards.build()));
            return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            fail();
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) {
            countDownLatch.countDown();
        }
    });
    countDownLatch.await();
    logger.info("--> try deleting the orphan snapshot");
    assertAcked(client.admin().cluster().prepareDeleteSnapshot(repositoryName, snapshotName).get("10s"));
}

Example 9 with SnapshotsInProgress

use of org.elasticsearch.cluster.SnapshotsInProgress in project elasticsearch by elastic.

the class AbstractSnapshotIntegTestCase method waitForCompletion.

public SnapshotInfo waitForCompletion(String repository, String snapshotName, TimeValue timeout) throws InterruptedException {
    long start = System.currentTimeMillis();
    while (System.currentTimeMillis() - start < timeout.millis()) {
        List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots(repository).setSnapshots(snapshotName).get().getSnapshots();
        assertThat(snapshotInfos.size(), equalTo(1));
        if (snapshotInfos.get(0).state().completed()) {
            // Make sure that snapshot clean up operations are finished
            ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get();
            SnapshotsInProgress snapshotsInProgress = stateResponse.getState().custom(SnapshotsInProgress.TYPE);
            if (snapshotsInProgress == null) {
                return snapshotInfos.get(0);
            } else {
                boolean found = false;
                for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
                    final Snapshot curr = entry.snapshot();
                    if (curr.getRepository().equals(repository) && curr.getSnapshotId().getName().equals(snapshotName)) {
                        found = true;
                        break;
                    }
                }
                if (found == false) {
                    return snapshotInfos.get(0);
                }
            }
        }
        Thread.sleep(100);
    }
    fail("Timeout!!!");
    return null;
}

Example 10 with SnapshotsInProgress

use of org.elasticsearch.cluster.SnapshotsInProgress in project elasticsearch by elastic.

the class SnapshotShardsService method processIndexShardSnapshots.

/**
     * Checks if any new shards should be snapshotted on this node
     *
     * @param event cluster state changed event
     */
private void processIndexShardSnapshots(ClusterChangedEvent event) {
    SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE);
    Map<Snapshot, SnapshotShards> survivors = new HashMap<>();
    // First, remove snapshots that are no longer there
    for (Map.Entry<Snapshot, SnapshotShards> entry : shardSnapshots.entrySet()) {
        final Snapshot snapshot = entry.getKey();
        if (snapshotsInProgress != null && snapshotsInProgress.snapshot(snapshot) != null) {
            survivors.put(entry.getKey(), entry.getValue());
        } else {
            // state update, which is being processed here
            for (IndexShardSnapshotStatus snapshotStatus : entry.getValue().shards.values()) {
                if (snapshotStatus.stage() == Stage.INIT || snapshotStatus.stage() == Stage.STARTED) {
                    snapshotStatus.abort();
                }
            }
        }
    }
    // For now we will be mostly dealing with a single snapshot at a time but might have multiple simultaneously running
    // snapshots in the future
    Map<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> newSnapshots = new HashMap<>();
    // Now go through all snapshots and update existing or create missing
    final String localNodeId = event.state().nodes().getLocalNodeId();
    final DiscoveryNode masterNode = event.state().nodes().getMasterNode();
    final Map<Snapshot, Map<String, IndexId>> snapshotIndices = new HashMap<>();
    if (snapshotsInProgress != null) {
        for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
            snapshotIndices.put(entry.snapshot(), entry.indices().stream().collect(Collectors.toMap(IndexId::getName, Function.identity())));
            if (entry.state() == State.STARTED) {
                Map<ShardId, IndexShardSnapshotStatus> startedShards = new HashMap<>();
                SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shard : entry.shards()) {
                    // Add all new shards to start processing on
                    if (localNodeId.equals(shard.value.nodeId())) {
                        if (shard.value.state() == State.INIT && (snapshotShards == null || !snapshotShards.shards.containsKey(shard.key))) {
                            logger.trace("[{}] - Adding shard to the queue", shard.key);
                            startedShards.put(shard.key, new IndexShardSnapshotStatus());
                        }
                    }
                }
                if (!startedShards.isEmpty()) {
                    newSnapshots.put(entry.snapshot(), startedShards);
                    if (snapshotShards != null) {
                        // We already saw this snapshot but we need to add more started shards
                        Map<ShardId, IndexShardSnapshotStatus> shards = new HashMap<>();
                        // Put all shards that were already running on this node
                        shards.putAll(snapshotShards.shards);
                        // Put all newly started shards
                        shards.putAll(startedShards);
                        survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(shards)));
                    } else {
                        // Brand new snapshot that we haven't seen before
                        survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(startedShards)));
                    }
                }
            } else if (entry.state() == State.ABORTED) {
                // Abort all running shards for this snapshot
                SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                if (snapshotShards != null) {
                    for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shard : entry.shards()) {
                        IndexShardSnapshotStatus snapshotStatus = snapshotShards.shards.get(shard.key);
                        if (snapshotStatus != null) {
                            switch(snapshotStatus.stage()) {
                                case INIT:
                                case STARTED:
                                    snapshotStatus.abort();
                                    break;
                                case FINALIZE:
                                    logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, letting it finish", entry.snapshot(), shard.key);
                                    break;
                                case DONE:
                                    logger.debug("[{}] trying to cancel snapshot on the shard [{}] that is already done, updating status on the master", entry.snapshot(), shard.key);
                                    updateIndexShardSnapshotStatus(entry.snapshot(), shard.key, new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                                    break;
                                case FAILURE:
                                    logger.debug("[{}] trying to cancel snapshot on the shard [{}] that has already failed, updating status on the master", entry.snapshot(), shard.key);
                                    updateIndexShardSnapshotStatus(entry.snapshot(), shard.key, new ShardSnapshotStatus(localNodeId, State.FAILED, snapshotStatus.failure()), masterNode);
                                    break;
                                default:
                                    throw new IllegalStateException("Unknown snapshot shard stage " + snapshotStatus.stage());
                            }
                        }
                    }
                }
            }
        }
    }
    // Update the list of snapshots that we saw and tried to started
    // If startup of these shards fails later, we don't want to try starting these shards again
    shutdownLock.lock();
    try {
        shardSnapshots = unmodifiableMap(survivors);
        if (shardSnapshots.isEmpty()) {
            // Notify all waiting threads that no more snapshots
            shutdownCondition.signalAll();
        }
    } finally {
        shutdownLock.unlock();
    }
    // We have new shards to starts
    if (newSnapshots.isEmpty() == false) {
        Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
        for (final Map.Entry<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> entry : newSnapshots.entrySet()) {
            Map<String, IndexId> indicesMap = snapshotIndices.get(entry.getKey());
            assert indicesMap != null;
            for (final Map.Entry<ShardId, IndexShardSnapshotStatus> shardEntry : entry.getValue().entrySet()) {
                final ShardId shardId = shardEntry.getKey();
                try {
                    final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id());
                    final IndexId indexId = indicesMap.get(shardId.getIndexName());
                    assert indexId != null;
                    executor.execute(new AbstractRunnable() {

                        @Override
                        public void doRun() {
                            snapshot(indexShard, entry.getKey(), indexId, shardEntry.getValue());
                            updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                        }

                        @Override
                        public void onFailure(Exception e) {
                            logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] [{}] failed to create snapshot", shardId, entry.getKey()), e);
                            updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.FAILED, ExceptionsHelper.detailedMessage(e)), masterNode);
                        }
                    });
                } catch (Exception e) {
                    updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.FAILED, ExceptionsHelper.detailedMessage(e)), masterNode);
                }
            }
        }
    }
}