Examples with IndexShardSnapshotStatus org.elasticsearch.index.snapshots.IndexShardSnapshotStatus used on opensource projects

Example 1 with IndexShardSnapshotStatus

use of org.elasticsearch.index.snapshots.IndexShardSnapshotStatus in project elasticsearch by elastic.

the class TransportSnapshotsStatusAction method buildResponse.

private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, List<SnapshotsInProgress.Entry> currentSnapshotEntries, TransportNodesSnapshotsStatus.NodesSnapshotStatus nodeSnapshotStatuses) throws IOException {
    // First process snapshot that are currently processed
    List<SnapshotStatus> builder = new ArrayList<>();
    Set<String> currentSnapshotNames = new HashSet<>();
    if (!currentSnapshotEntries.isEmpty()) {
        Map<String, TransportNodesSnapshotsStatus.NodeSnapshotStatus> nodeSnapshotStatusMap;
        if (nodeSnapshotStatuses != null) {
            nodeSnapshotStatusMap = nodeSnapshotStatuses.getNodesMap();
        } else {
            nodeSnapshotStatusMap = new HashMap<>();
        }
        for (SnapshotsInProgress.Entry entry : currentSnapshotEntries) {
            currentSnapshotNames.add(entry.snapshot().getSnapshotId().getName());
            List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
            for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shardEntry : entry.shards()) {
                SnapshotsInProgress.ShardSnapshotStatus status = shardEntry.value;
                if (status.nodeId() != null) {
                    // We should have information about this shard from the shard:
                    TransportNodesSnapshotsStatus.NodeSnapshotStatus nodeStatus = nodeSnapshotStatusMap.get(status.nodeId());
                    if (nodeStatus != null) {
                        Map<ShardId, SnapshotIndexShardStatus> shardStatues = nodeStatus.status().get(entry.snapshot());
                        if (shardStatues != null) {
                            SnapshotIndexShardStatus shardStatus = shardStatues.get(shardEntry.key);
                            if (shardStatus != null) {
                                // We have full information about this shard
                                shardStatusBuilder.add(shardStatus);
                                continue;
                            }
                        }
                    }
                }
                final SnapshotIndexShardStage stage;
                switch(shardEntry.value.state()) {
                    case FAILED:
                    case ABORTED:
                    case MISSING:
                        stage = SnapshotIndexShardStage.FAILURE;
                        break;
                    case INIT:
                    case WAITING:
                    case STARTED:
                        stage = SnapshotIndexShardStage.STARTED;
                        break;
                    case SUCCESS:
                        stage = SnapshotIndexShardStage.DONE;
                        break;
                    default:
                        throw new IllegalArgumentException("Unknown snapshot state " + shardEntry.value.state());
                }
                SnapshotIndexShardStatus shardStatus = new SnapshotIndexShardStatus(shardEntry.key, stage);
                shardStatusBuilder.add(shardStatus);
            }
            builder.add(new SnapshotStatus(entry.snapshot(), entry.state(), Collections.unmodifiableList(shardStatusBuilder)));
        }
    }
    // Now add snapshots on disk that are not currently running
    final String repositoryName = request.repository();
    if (Strings.hasText(repositoryName) && request.snapshots() != null && request.snapshots().length > 0) {
        final Set<String> requestedSnapshotNames = Sets.newHashSet(request.snapshots());
        final RepositoryData repositoryData = snapshotsService.getRepositoryData(repositoryName);
        final Map<String, SnapshotId> matchedSnapshotIds = repositoryData.getAllSnapshotIds().stream().filter(s -> requestedSnapshotNames.contains(s.getName())).collect(Collectors.toMap(SnapshotId::getName, Function.identity()));
        for (final String snapshotName : request.snapshots()) {
            if (currentSnapshotNames.contains(snapshotName)) {
                // we've already found this snapshot in the current snapshot entries, so skip over
                continue;
            }
            SnapshotId snapshotId = matchedSnapshotIds.get(snapshotName);
            if (snapshotId == null) {
                // neither in the current snapshot entries nor found in the repository
                if (request.ignoreUnavailable()) {
                    // ignoring unavailable snapshots, so skip over
                    logger.debug("snapshot status request ignoring snapshot [{}], not found in repository [{}]", snapshotName, repositoryName);
                    continue;
                } else {
                    throw new SnapshotMissingException(repositoryName, snapshotName);
                }
            } else if (repositoryData.getIncompatibleSnapshotIds().contains(snapshotId)) {
                throw new SnapshotException(repositoryName, snapshotName, "cannot get the status for an incompatible snapshot");
            }
            SnapshotInfo snapshotInfo = snapshotsService.snapshot(repositoryName, snapshotId);
            List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
            if (snapshotInfo.state().completed()) {
                Map<ShardId, IndexShardSnapshotStatus> shardStatues = snapshotsService.snapshotShards(request.repository(), snapshotInfo);
                for (Map.Entry<ShardId, IndexShardSnapshotStatus> shardStatus : shardStatues.entrySet()) {
                    shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), shardStatus.getValue()));
                }
                final SnapshotsInProgress.State state;
                switch(snapshotInfo.state()) {
                    case FAILED:
                        state = SnapshotsInProgress.State.FAILED;
                        break;
                    case SUCCESS:
                    case PARTIAL:
                        // Translating both PARTIAL and SUCCESS to SUCCESS for now
                        // TODO: add the differentiation on the metadata level in the next major release
                        state = SnapshotsInProgress.State.SUCCESS;
                        break;
                    default:
                        throw new IllegalArgumentException("Unknown snapshot state " + snapshotInfo.state());
                }
                builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotId), state, Collections.unmodifiableList(shardStatusBuilder)));
            }
        }
    }
    return new SnapshotsStatusResponse(Collections.unmodifiableList(builder));
}

Example 2 with IndexShardSnapshotStatus

use of org.elasticsearch.index.snapshots.IndexShardSnapshotStatus in project elasticsearch by elastic.

the class SnapshotShardsService method syncShardStatsOnNewMaster.

/**
     * Checks if any shards were processed that the new master doesn't know about
     */
private void syncShardStatsOnNewMaster(ClusterChangedEvent event) {
    SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE);
    if (snapshotsInProgress == null) {
        return;
    }
    final String localNodeId = event.state().nodes().getLocalNodeId();
    final DiscoveryNode masterNode = event.state().nodes().getMasterNode();
    for (SnapshotsInProgress.Entry snapshot : snapshotsInProgress.entries()) {
        if (snapshot.state() == State.STARTED || snapshot.state() == State.ABORTED) {
            Map<ShardId, IndexShardSnapshotStatus> localShards = currentSnapshotShards(snapshot.snapshot());
            if (localShards != null) {
                ImmutableOpenMap<ShardId, ShardSnapshotStatus> masterShards = snapshot.shards();
                for (Map.Entry<ShardId, IndexShardSnapshotStatus> localShard : localShards.entrySet()) {
                    ShardId shardId = localShard.getKey();
                    IndexShardSnapshotStatus localShardStatus = localShard.getValue();
                    ShardSnapshotStatus masterShard = masterShards.get(shardId);
                    if (masterShard != null && masterShard.state().completed() == false) {
                        // Master knows about the shard and thinks it has not completed
                        if (localShardStatus.stage() == Stage.DONE) {
                            // but we think the shard is done - we need to make new master know that the shard is done
                            logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard is done locally, updating status on the master", snapshot.snapshot(), shardId);
                            updateIndexShardSnapshotStatus(snapshot.snapshot(), shardId, new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                        } else if (localShard.getValue().stage() == Stage.FAILURE) {
                            // but we think the shard failed - we need to make new master know that the shard failed
                            logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard failed locally, updating status on master", snapshot.snapshot(), shardId);
                            updateIndexShardSnapshotStatus(snapshot.snapshot(), shardId, new ShardSnapshotStatus(localNodeId, State.FAILED, localShardStatus.failure()), masterNode);
                        }
                    }
                }
            }
        }
    }
}

Example 3 with IndexShardSnapshotStatus

use of org.elasticsearch.index.snapshots.IndexShardSnapshotStatus in project elasticsearch by elastic.

the class BlobStoreRepository method getShardSnapshotStatus.

@Override
public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) {
    Context context = new Context(snapshotId, version, indexId, shardId);
    BlobStoreIndexShardSnapshot snapshot = context.loadSnapshot();
    IndexShardSnapshotStatus status = new IndexShardSnapshotStatus();
    status.updateStage(IndexShardSnapshotStatus.Stage.DONE);
    status.startTime(snapshot.startTime());
    status.files(snapshot.numberOfFiles(), snapshot.totalSize());
    // The snapshot is done which means the number of processed files is the same as total
    status.processedFiles(snapshot.numberOfFiles(), snapshot.totalSize());
    status.time(snapshot.time());
    return status;
}

Example 4 with IndexShardSnapshotStatus

use of org.elasticsearch.index.snapshots.IndexShardSnapshotStatus in project elasticsearch by elastic.

the class TransportNodesSnapshotsStatus method nodeOperation.

@Override
protected NodeSnapshotStatus nodeOperation(NodeRequest request) {
    Map<Snapshot, Map<ShardId, SnapshotIndexShardStatus>> snapshotMapBuilder = new HashMap<>();
    try {
        String nodeId = clusterService.localNode().getId();
        for (Snapshot snapshot : request.snapshots) {
            Map<ShardId, IndexShardSnapshotStatus> shardsStatus = snapshotShardsService.currentSnapshotShards(snapshot);
            if (shardsStatus == null) {
                continue;
            }
            Map<ShardId, SnapshotIndexShardStatus> shardMapBuilder = new HashMap<>();
            for (Map.Entry<ShardId, IndexShardSnapshotStatus> shardEntry : shardsStatus.entrySet()) {
                SnapshotIndexShardStatus shardStatus;
                IndexShardSnapshotStatus.Stage stage = shardEntry.getValue().stage();
                if (stage != IndexShardSnapshotStatus.Stage.DONE && stage != IndexShardSnapshotStatus.Stage.FAILURE) {
                    // Store node id for the snapshots that are currently running.
                    shardStatus = new SnapshotIndexShardStatus(shardEntry.getKey(), shardEntry.getValue(), nodeId);
                } else {
                    shardStatus = new SnapshotIndexShardStatus(shardEntry.getKey(), shardEntry.getValue());
                }
                shardMapBuilder.put(shardEntry.getKey(), shardStatus);
            }
            snapshotMapBuilder.put(snapshot, unmodifiableMap(shardMapBuilder));
        }
        return new NodeSnapshotStatus(clusterService.localNode(), unmodifiableMap(snapshotMapBuilder));
    } catch (Exception e) {
        throw new ElasticsearchException("failed to load metadata", e);
    }
}

Example 5 with IndexShardSnapshotStatus

use of org.elasticsearch.index.snapshots.IndexShardSnapshotStatus in project elasticsearch by elastic.

the class SnapshotShardsService method processIndexShardSnapshots.

/**
     * Checks if any new shards should be snapshotted on this node
     *
     * @param event cluster state changed event
     */
private void processIndexShardSnapshots(ClusterChangedEvent event) {
    SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE);
    Map<Snapshot, SnapshotShards> survivors = new HashMap<>();
    // First, remove snapshots that are no longer there
    for (Map.Entry<Snapshot, SnapshotShards> entry : shardSnapshots.entrySet()) {
        final Snapshot snapshot = entry.getKey();
        if (snapshotsInProgress != null && snapshotsInProgress.snapshot(snapshot) != null) {
            survivors.put(entry.getKey(), entry.getValue());
        } else {
            // state update, which is being processed here
            for (IndexShardSnapshotStatus snapshotStatus : entry.getValue().shards.values()) {
                if (snapshotStatus.stage() == Stage.INIT || snapshotStatus.stage() == Stage.STARTED) {
                    snapshotStatus.abort();
                }
            }
        }
    }
    // For now we will be mostly dealing with a single snapshot at a time but might have multiple simultaneously running
    // snapshots in the future
    Map<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> newSnapshots = new HashMap<>();
    // Now go through all snapshots and update existing or create missing
    final String localNodeId = event.state().nodes().getLocalNodeId();
    final DiscoveryNode masterNode = event.state().nodes().getMasterNode();
    final Map<Snapshot, Map<String, IndexId>> snapshotIndices = new HashMap<>();
    if (snapshotsInProgress != null) {
        for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
            snapshotIndices.put(entry.snapshot(), entry.indices().stream().collect(Collectors.toMap(IndexId::getName, Function.identity())));
            if (entry.state() == State.STARTED) {
                Map<ShardId, IndexShardSnapshotStatus> startedShards = new HashMap<>();
                SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shard : entry.shards()) {
                    // Add all new shards to start processing on
                    if (localNodeId.equals(shard.value.nodeId())) {
                        if (shard.value.state() == State.INIT && (snapshotShards == null || !snapshotShards.shards.containsKey(shard.key))) {
                            logger.trace("[{}] - Adding shard to the queue", shard.key);
                            startedShards.put(shard.key, new IndexShardSnapshotStatus());
                        }
                    }
                }
                if (!startedShards.isEmpty()) {
                    newSnapshots.put(entry.snapshot(), startedShards);
                    if (snapshotShards != null) {
                        // We already saw this snapshot but we need to add more started shards
                        Map<ShardId, IndexShardSnapshotStatus> shards = new HashMap<>();
                        // Put all shards that were already running on this node
                        shards.putAll(snapshotShards.shards);
                        // Put all newly started shards
                        shards.putAll(startedShards);
                        survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(shards)));
                    } else {
                        // Brand new snapshot that we haven't seen before
                        survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(startedShards)));
                    }
                }
            } else if (entry.state() == State.ABORTED) {
                // Abort all running shards for this snapshot
                SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                if (snapshotShards != null) {
                    for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shard : entry.shards()) {
                        IndexShardSnapshotStatus snapshotStatus = snapshotShards.shards.get(shard.key);
                        if (snapshotStatus != null) {
                            switch(snapshotStatus.stage()) {
                                case INIT:
                                case STARTED:
                                    snapshotStatus.abort();
                                    break;
                                case FINALIZE:
                                    logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, letting it finish", entry.snapshot(), shard.key);
                                    break;
                                case DONE:
                                    logger.debug("[{}] trying to cancel snapshot on the shard [{}] that is already done, updating status on the master", entry.snapshot(), shard.key);
                                    updateIndexShardSnapshotStatus(entry.snapshot(), shard.key, new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                                    break;
                                case FAILURE:
                                    logger.debug("[{}] trying to cancel snapshot on the shard [{}] that has already failed, updating status on the master", entry.snapshot(), shard.key);
                                    updateIndexShardSnapshotStatus(entry.snapshot(), shard.key, new ShardSnapshotStatus(localNodeId, State.FAILED, snapshotStatus.failure()), masterNode);
                                    break;
                                default:
                                    throw new IllegalStateException("Unknown snapshot shard stage " + snapshotStatus.stage());
                            }
                        }
                    }
                }
            }
        }
    }
    // Update the list of snapshots that we saw and tried to started
    // If startup of these shards fails later, we don't want to try starting these shards again
    shutdownLock.lock();
    try {
        shardSnapshots = unmodifiableMap(survivors);
        if (shardSnapshots.isEmpty()) {
            // Notify all waiting threads that no more snapshots
            shutdownCondition.signalAll();
        }
    } finally {
        shutdownLock.unlock();
    }
    // We have new shards to starts
    if (newSnapshots.isEmpty() == false) {
        Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
        for (final Map.Entry<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> entry : newSnapshots.entrySet()) {
            Map<String, IndexId> indicesMap = snapshotIndices.get(entry.getKey());
            assert indicesMap != null;
            for (final Map.Entry<ShardId, IndexShardSnapshotStatus> shardEntry : entry.getValue().entrySet()) {
                final ShardId shardId = shardEntry.getKey();
                try {
                    final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id());
                    final IndexId indexId = indicesMap.get(shardId.getIndexName());
                    assert indexId != null;
                    executor.execute(new AbstractRunnable() {

                        @Override
                        public void doRun() {
                            snapshot(indexShard, entry.getKey(), indexId, shardEntry.getValue());
                            updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                        }

                        @Override
                        public void onFailure(Exception e) {
                            logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] [{}] failed to create snapshot", shardId, entry.getKey()), e);
                            updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.FAILED, ExceptionsHelper.detailedMessage(e)), masterNode);
                        }
                    });
                } catch (Exception e) {
                    updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.FAILED, ExceptionsHelper.detailedMessage(e)), masterNode);
                }
            }
        }
    }
}