Examples with Repository org.elasticsearch.repositories.Repository used on opensource projects

Example 21 with Repository

use of org.elasticsearch.repositories.Repository in project elasticsearch by elastic.

the class IndexShard method startRecovery.

public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService, PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService, BiConsumer<String, MappingMetaData> mappingUpdateConsumer, IndicesService indicesService) {
    // }
    assert recoveryState.getRecoverySource().equals(shardRouting.recoverySource());
    switch(recoveryState.getRecoverySource().getType()) {
        case EMPTY_STORE:
        case EXISTING_STORE:
            // mark the shard as recovering on the cluster state thread
            markAsRecovering("from store", recoveryState);
            threadPool.generic().execute(() -> {
                try {
                    if (recoverFromStore()) {
                        recoveryListener.onRecoveryDone(recoveryState);
                    }
                } catch (Exception e) {
                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                }
            });
            break;
        case PEER:
            try {
                markAsRecovering("from " + recoveryState.getSourceNode(), recoveryState);
                recoveryTargetService.startRecovery(this, recoveryState.getSourceNode(), recoveryListener);
            } catch (Exception e) {
                failShard("corrupted preexisting index", e);
                recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
            }
            break;
        case SNAPSHOT:
            // mark the shard as recovering on the cluster state thread
            markAsRecovering("from snapshot", recoveryState);
            SnapshotRecoverySource recoverySource = (SnapshotRecoverySource) recoveryState.getRecoverySource();
            threadPool.generic().execute(() -> {
                try {
                    final Repository repository = repositoriesService.repository(recoverySource.snapshot().getRepository());
                    if (restoreFromRepository(repository)) {
                        recoveryListener.onRecoveryDone(recoveryState);
                    }
                } catch (Exception e) {
                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                }
            });
            break;
        case LOCAL_SHARDS:
            final IndexMetaData indexMetaData = indexSettings().getIndexMetaData();
            final Index mergeSourceIndex = indexMetaData.getMergeSourceIndex();
            final List<IndexShard> startedShards = new ArrayList<>();
            final IndexService sourceIndexService = indicesService.indexService(mergeSourceIndex);
            final int numShards = sourceIndexService != null ? sourceIndexService.getIndexSettings().getNumberOfShards() : -1;
            if (sourceIndexService != null) {
                for (IndexShard shard : sourceIndexService) {
                    if (shard.state() == IndexShardState.STARTED) {
                        startedShards.add(shard);
                    }
                }
            }
            if (numShards == startedShards.size()) {
                // mark the shard as recovering on the cluster state thread
                markAsRecovering("from local shards", recoveryState);
                threadPool.generic().execute(() -> {
                    try {
                        final Set<ShardId> shards = IndexMetaData.selectShrinkShards(shardId().id(), sourceIndexService.getMetaData(), +indexMetaData.getNumberOfShards());
                        if (recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream().filter((s) -> shards.contains(s.shardId())).collect(Collectors.toList()))) {
                            recoveryListener.onRecoveryDone(recoveryState);
                        }
                    } catch (Exception e) {
                        recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                    }
                });
            } else {
                final RuntimeException e;
                if (numShards == -1) {
                    e = new IndexNotFoundException(mergeSourceIndex);
                } else {
                    e = new IllegalStateException("not all shards from index " + mergeSourceIndex + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard " + shardId());
                }
                throw e;
            }
            break;
        default:
            throw new IllegalArgumentException("Unknown recovery source " + recoveryState.getRecoverySource());
    }
}

Example 22 with Repository

use of org.elasticsearch.repositories.Repository in project elasticsearch by elastic.

the class SnapshotsService method snapshotShards.

/**
     * Returns status of shards  currently finished snapshots
     * <p>
     * This method is executed on master node and it's complimentary to the {@link SnapshotShardsService#currentSnapshotShards(Snapshot)} because it
     * returns similar information but for already finished snapshots.
     * </p>
     *
     * @param repositoryName  repository name
     * @param snapshotInfo    snapshot info
     * @return map of shard id to snapshot status
     */
public Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String repositoryName, final SnapshotInfo snapshotInfo) throws IOException {
    Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
    Repository repository = repositoriesService.repository(repositoryName);
    RepositoryData repositoryData = repository.getRepositoryData();
    MetaData metaData = repository.getSnapshotMetaData(snapshotInfo, repositoryData.resolveIndices(snapshotInfo.indices()));
    for (String index : snapshotInfo.indices()) {
        IndexId indexId = repositoryData.resolveIndexId(index);
        IndexMetaData indexMetaData = metaData.indices().get(index);
        if (indexMetaData != null) {
            int numberOfShards = indexMetaData.getNumberOfShards();
            for (int i = 0; i < numberOfShards; i++) {
                ShardId shardId = new ShardId(indexMetaData.getIndex(), i);
                SnapshotShardFailure shardFailure = findShardFailure(snapshotInfo.shardFailures(), shardId);
                if (shardFailure != null) {
                    IndexShardSnapshotStatus shardSnapshotStatus = new IndexShardSnapshotStatus();
                    shardSnapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE);
                    shardSnapshotStatus.failure(shardFailure.reason());
                    shardStatus.put(shardId, shardSnapshotStatus);
                } else {
                    IndexShardSnapshotStatus shardSnapshotStatus = repository.getShardSnapshotStatus(snapshotInfo.snapshotId(), snapshotInfo.version(), indexId, shardId);
                    shardStatus.put(shardId, shardSnapshotStatus);
                }
            }
        }
    }
    return unmodifiableMap(shardStatus);
}

Example 23 with Repository

use of org.elasticsearch.repositories.Repository in project elasticsearch by elastic.

the class SnapshotsService method deleteSnapshot.

/**
     * Deletes a snapshot from the repository, looking up the {@link Snapshot} reference before deleting.
     * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
     *
     * @param repositoryName  repositoryName
     * @param snapshotName    snapshotName
     * @param listener        listener
     */
public void deleteSnapshot(final String repositoryName, final String snapshotName, final DeleteSnapshotListener listener, final boolean immediatePriority) {
    // First, look for the snapshot in the repository
    final Repository repository = repositoriesService.repository(repositoryName);
    final RepositoryData repositoryData = repository.getRepositoryData();
    final Optional<SnapshotId> incompatibleSnapshotId = repositoryData.getIncompatibleSnapshotIds().stream().filter(s -> snapshotName.equals(s.getName())).findFirst();
    if (incompatibleSnapshotId.isPresent()) {
        throw new SnapshotException(repositoryName, snapshotName, "cannot delete incompatible snapshot");
    }
    Optional<SnapshotId> matchedEntry = repositoryData.getSnapshotIds().stream().filter(s -> s.getName().equals(snapshotName)).findFirst();
    // if nothing found by the same name, then look in the cluster state for current in progress snapshots
    if (matchedEntry.isPresent() == false) {
        matchedEntry = currentSnapshots(repositoryName, Collections.emptyList()).stream().map(e -> e.snapshot().getSnapshotId()).filter(s -> s.getName().equals(snapshotName)).findFirst();
    }
    if (matchedEntry.isPresent() == false) {
        throw new SnapshotMissingException(repositoryName, snapshotName);
    }
    deleteSnapshot(new Snapshot(repositoryName, matchedEntry.get()), listener, repositoryData.getGenId(), immediatePriority);
}

Example 24 with Repository

use of org.elasticsearch.repositories.Repository in project crate by crate.

the class SnapshotRestoreIntegrationTest method getRepositoryData.

private RepositoryData getRepositoryData() throws Exception {
    RepositoriesService service = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    Repository repository = service.repository(REPOSITORY_NAME);
    ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, internalCluster().getMasterName());
    final SetOnce<RepositoryData> repositoryData = new SetOnce<>();
    final CountDownLatch latch = new CountDownLatch(1);
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
        repositoryData.set(ESBlobStoreTestCase.getRepositoryData(repository));
        latch.countDown();
    });
    latch.await();
    return repositoryData.get();
}

Example 25 with Repository

use of org.elasticsearch.repositories.Repository in project crate by crate.

the class SnapshotShardsService method snapshot.

/**
 * Creates shard snapshot
 *
 * @param snapshot       snapshot
 * @param snapshotStatus snapshot status
 */
private void snapshot(final ShardId shardId, final Snapshot snapshot, final IndexId indexId, final IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, ActionListener<String> listener) {
    try {
        final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id());
        if (indexShard.routingEntry().primary() == false) {
            throw new IndexShardSnapshotFailedException(shardId, "snapshot should be performed only on primary");
        }
        if (indexShard.routingEntry().relocating()) {
            // do not snapshot when in the process of relocation of primaries so we won't get conflicts
            throw new IndexShardSnapshotFailedException(shardId, "cannot snapshot while relocating");
        }
        final IndexShardState indexShardState = indexShard.state();
        if (indexShardState == IndexShardState.CREATED || indexShardState == IndexShardState.RECOVERING) {
            // shard has just been created, or still recovering
            throw new IndexShardSnapshotFailedException(shardId, "shard didn't fully recover yet");
        }
        final Repository repository = repositoriesService.repository(snapshot.getRepository());
        Engine.IndexCommitRef snapshotRef = null;
        try {
            // we flush first to make sure we get the latest writes snapshotted
            snapshotRef = indexShard.acquireLastIndexCommit(true);
            repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId, snapshotRef.getIndexCommit(), snapshotStatus, writeShardGens, ActionListener.runBefore(listener, snapshotRef::close));
        } catch (Exception e) {
            IOUtils.close(snapshotRef);
            throw e;
        }
    } catch (Exception e) {
        listener.onFailure(e);
    }
}