Examples with Repository org.elasticsearch.repositories.Repository used on opensource projects

Example 6 with Repository

use of org.elasticsearch.repositories.Repository in project elasticsearch by elastic.

the class SnapshotsService method beginSnapshot.

/**
     * Starts snapshot.
     * <p>
     * Creates snapshot in repository and updates snapshot metadata record with list of shards that needs to be processed.
     *
     * @param clusterState               cluster state
     * @param snapshot                   snapshot meta data
     * @param partial                    allow partial snapshots
     * @param userCreateSnapshotListener listener
     */
private void beginSnapshot(final ClusterState clusterState, final SnapshotsInProgress.Entry snapshot, final boolean partial, final CreateSnapshotListener userCreateSnapshotListener) {
    boolean snapshotCreated = false;
    try {
        Repository repository = repositoriesService.repository(snapshot.snapshot().getRepository());
        MetaData metaData = clusterState.metaData();
        if (!snapshot.includeGlobalState()) {
            // Remove global state from the cluster state
            MetaData.Builder builder = MetaData.builder();
            for (IndexId index : snapshot.indices()) {
                builder.put(metaData.index(index.getName()), false);
            }
            metaData = builder.build();
        }
        repository.initializeSnapshot(snapshot.snapshot().getSnapshotId(), snapshot.indices(), metaData);
        snapshotCreated = true;
        if (snapshot.indices().isEmpty()) {
            // No indices in this snapshot - we are done
            userCreateSnapshotListener.onResponse();
            endSnapshot(snapshot);
            return;
        }
        clusterService.submitStateUpdateTask("update_snapshot [" + snapshot.snapshot() + "]", new ClusterStateUpdateTask() {

            boolean accepted = false;

            SnapshotsInProgress.Entry updatedSnapshot;

            String failure = null;

            @Override
            public ClusterState execute(ClusterState currentState) {
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
                    if (entry.snapshot().equals(snapshot.snapshot())) {
                        // Replace the snapshot that was just created
                        ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = shards(currentState, entry.indices());
                        if (!partial) {
                            Tuple<Set<String>, Set<String>> indicesWithMissingShards = indicesWithMissingShards(shards, currentState.metaData());
                            Set<String> missing = indicesWithMissingShards.v1();
                            Set<String> closed = indicesWithMissingShards.v2();
                            if (missing.isEmpty() == false || closed.isEmpty() == false) {
                                StringBuilder failureMessage = new StringBuilder();
                                updatedSnapshot = new SnapshotsInProgress.Entry(entry, State.FAILED, shards);
                                entries.add(updatedSnapshot);
                                if (missing.isEmpty() == false) {
                                    failureMessage.append("Indices don't have primary shards ");
                                    failureMessage.append(missing);
                                }
                                if (closed.isEmpty() == false) {
                                    if (failureMessage.length() > 0) {
                                        failureMessage.append("; ");
                                    }
                                    failureMessage.append("Indices are closed ");
                                    failureMessage.append(closed);
                                }
                                failure = failureMessage.toString();
                                continue;
                            }
                        }
                        updatedSnapshot = new SnapshotsInProgress.Entry(entry, State.STARTED, shards);
                        entries.add(updatedSnapshot);
                        if (!completed(shards.values())) {
                            accepted = true;
                        }
                    } else {
                        entries.add(entry);
                    }
                }
                return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to create snapshot", snapshot.snapshot().getSnapshotId()), e);
                removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, e));
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                // The userCreateSnapshotListener.onResponse() notifies caller that the snapshot was accepted
                // for processing. If client wants to wait for the snapshot completion, it can register snapshot
                // completion listener in this method. For the snapshot completion to work properly, the snapshot
                // should still exist when listener is registered.
                userCreateSnapshotListener.onResponse();
                // go ahead and continue working on this snapshot rather then end here.
                if (!accepted && updatedSnapshot != null) {
                    endSnapshot(updatedSnapshot, failure);
                }
            }
        });
    } catch (Exception e) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to create snapshot [{}]", snapshot.snapshot().getSnapshotId()), e);
        removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, e));
    }
}

Example 7 with Repository

use of org.elasticsearch.repositories.Repository in project elasticsearch by elastic.

the class IndexShard method startRecovery.

public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService, PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService, BiConsumer<String, MappingMetaData> mappingUpdateConsumer, IndicesService indicesService) {
    // }
    assert recoveryState.getRecoverySource().equals(shardRouting.recoverySource());
    switch(recoveryState.getRecoverySource().getType()) {
        case EMPTY_STORE:
        case EXISTING_STORE:
            // mark the shard as recovering on the cluster state thread
            markAsRecovering("from store", recoveryState);
            threadPool.generic().execute(() -> {
                try {
                    if (recoverFromStore()) {
                        recoveryListener.onRecoveryDone(recoveryState);
                    }
                } catch (Exception e) {
                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                }
            });
            break;
        case PEER:
            try {
                markAsRecovering("from " + recoveryState.getSourceNode(), recoveryState);
                recoveryTargetService.startRecovery(this, recoveryState.getSourceNode(), recoveryListener);
            } catch (Exception e) {
                failShard("corrupted preexisting index", e);
                recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
            }
            break;
        case SNAPSHOT:
            // mark the shard as recovering on the cluster state thread
            markAsRecovering("from snapshot", recoveryState);
            SnapshotRecoverySource recoverySource = (SnapshotRecoverySource) recoveryState.getRecoverySource();
            threadPool.generic().execute(() -> {
                try {
                    final Repository repository = repositoriesService.repository(recoverySource.snapshot().getRepository());
                    if (restoreFromRepository(repository)) {
                        recoveryListener.onRecoveryDone(recoveryState);
                    }
                } catch (Exception e) {
                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                }
            });
            break;
        case LOCAL_SHARDS:
            final IndexMetaData indexMetaData = indexSettings().getIndexMetaData();
            final Index mergeSourceIndex = indexMetaData.getMergeSourceIndex();
            final List<IndexShard> startedShards = new ArrayList<>();
            final IndexService sourceIndexService = indicesService.indexService(mergeSourceIndex);
            final int numShards = sourceIndexService != null ? sourceIndexService.getIndexSettings().getNumberOfShards() : -1;
            if (sourceIndexService != null) {
                for (IndexShard shard : sourceIndexService) {
                    if (shard.state() == IndexShardState.STARTED) {
                        startedShards.add(shard);
                    }
                }
            }
            if (numShards == startedShards.size()) {
                // mark the shard as recovering on the cluster state thread
                markAsRecovering("from local shards", recoveryState);
                threadPool.generic().execute(() -> {
                    try {
                        final Set<ShardId> shards = IndexMetaData.selectShrinkShards(shardId().id(), sourceIndexService.getMetaData(), +indexMetaData.getNumberOfShards());
                        if (recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream().filter((s) -> shards.contains(s.shardId())).collect(Collectors.toList()))) {
                            recoveryListener.onRecoveryDone(recoveryState);
                        }
                    } catch (Exception e) {
                        recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                    }
                });
            } else {
                final RuntimeException e;
                if (numShards == -1) {
                    e = new IndexNotFoundException(mergeSourceIndex);
                } else {
                    e = new IllegalStateException("not all shards from index " + mergeSourceIndex + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard " + shardId());
                }
                throw e;
            }
            break;
        default:
            throw new IllegalArgumentException("Unknown recovery source " + recoveryState.getRecoverySource());
    }
}

Example 8 with Repository

use of org.elasticsearch.repositories.Repository in project elasticsearch by elastic.

the class SnapshotsService method snapshotShards.

/**
     * Returns status of shards  currently finished snapshots
     * <p>
     * This method is executed on master node and it's complimentary to the {@link SnapshotShardsService#currentSnapshotShards(Snapshot)} because it
     * returns similar information but for already finished snapshots.
     * </p>
     *
     * @param repositoryName  repository name
     * @param snapshotInfo    snapshot info
     * @return map of shard id to snapshot status
     */
public Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String repositoryName, final SnapshotInfo snapshotInfo) throws IOException {
    Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
    Repository repository = repositoriesService.repository(repositoryName);
    RepositoryData repositoryData = repository.getRepositoryData();
    MetaData metaData = repository.getSnapshotMetaData(snapshotInfo, repositoryData.resolveIndices(snapshotInfo.indices()));
    for (String index : snapshotInfo.indices()) {
        IndexId indexId = repositoryData.resolveIndexId(index);
        IndexMetaData indexMetaData = metaData.indices().get(index);
        if (indexMetaData != null) {
            int numberOfShards = indexMetaData.getNumberOfShards();
            for (int i = 0; i < numberOfShards; i++) {
                ShardId shardId = new ShardId(indexMetaData.getIndex(), i);
                SnapshotShardFailure shardFailure = findShardFailure(snapshotInfo.shardFailures(), shardId);
                if (shardFailure != null) {
                    IndexShardSnapshotStatus shardSnapshotStatus = new IndexShardSnapshotStatus();
                    shardSnapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE);
                    shardSnapshotStatus.failure(shardFailure.reason());
                    shardStatus.put(shardId, shardSnapshotStatus);
                } else {
                    IndexShardSnapshotStatus shardSnapshotStatus = repository.getShardSnapshotStatus(snapshotInfo.snapshotId(), snapshotInfo.version(), indexId, shardId);
                    shardStatus.put(shardId, shardSnapshotStatus);
                }
            }
        }
    }
    return unmodifiableMap(shardStatus);
}

Example 9 with Repository

use of org.elasticsearch.repositories.Repository in project elasticsearch by elastic.

the class SnapshotsService method deleteSnapshot.

/**
     * Deletes a snapshot from the repository, looking up the {@link Snapshot} reference before deleting.
     * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
     *
     * @param repositoryName  repositoryName
     * @param snapshotName    snapshotName
     * @param listener        listener
     */
public void deleteSnapshot(final String repositoryName, final String snapshotName, final DeleteSnapshotListener listener, final boolean immediatePriority) {
    // First, look for the snapshot in the repository
    final Repository repository = repositoriesService.repository(repositoryName);
    final RepositoryData repositoryData = repository.getRepositoryData();
    final Optional<SnapshotId> incompatibleSnapshotId = repositoryData.getIncompatibleSnapshotIds().stream().filter(s -> snapshotName.equals(s.getName())).findFirst();
    if (incompatibleSnapshotId.isPresent()) {
        throw new SnapshotException(repositoryName, snapshotName, "cannot delete incompatible snapshot");
    }
    Optional<SnapshotId> matchedEntry = repositoryData.getSnapshotIds().stream().filter(s -> s.getName().equals(snapshotName)).findFirst();
    // if nothing found by the same name, then look in the cluster state for current in progress snapshots
    if (matchedEntry.isPresent() == false) {
        matchedEntry = currentSnapshots(repositoryName, Collections.emptyList()).stream().map(e -> e.snapshot().getSnapshotId()).filter(s -> s.getName().equals(snapshotName)).findFirst();
    }
    if (matchedEntry.isPresent() == false) {
        throw new SnapshotMissingException(repositoryName, snapshotName);
    }
    deleteSnapshot(new Snapshot(repositoryName, matchedEntry.get()), listener, repositoryData.getGenId(), immediatePriority);
}