Examples with Snapshot org.elasticsearch.snapshots.Snapshot used on opensource projects

Example 21 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project crate by crate.

the class IndexRecoveryIT method testSnapshotRecovery.

@Test
public void testSnapshotRecovery() throws Exception {
    logger.info("--> start node A");
    String nodeA = internalCluster().startNode();
    logger.info("--> create repository");
    execute("CREATE REPOSITORY " + REPO_NAME + " TYPE FS WITH (location = '" + randomRepoPath() + "', compress=false)");
    ensureGreen();
    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> snapshot");
    var snapshotName = REPO_NAME + "." + SNAP_NAME;
    execute("CREATE SNAPSHOT " + snapshotName + " ALL WITH (wait_for_completion=true)");
    execute("SELECT state FROM sys.snapshots WHERE name = '" + SNAP_NAME + "'");
    assertThat(response.rows()[0][0], is("SUCCESS"));
    execute("ALTER TABLE " + INDEX_NAME + " CLOSE");
    logger.info("--> restore");
    execute("RESTORE SNAPSHOT " + snapshotName + " ALL WITH (wait_for_completion=true)");
    ensureGreen();
    var snapshotInfo = client().execute(GetSnapshotsAction.INSTANCE, new GetSnapshotsRequest(REPO_NAME, new String[] { SNAP_NAME })).get().getSnapshots().get(0);
    logger.info("--> request recoveries");
    var indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null);
    RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(indexName)).actionGet();
    for (Map.Entry<String, List<RecoveryState>> indexRecoveryStates : response.shardRecoveryStates().entrySet()) {
        assertThat(indexRecoveryStates.getKey(), equalTo(indexName));
        List<RecoveryState> recoveryStates = indexRecoveryStates.getValue();
        assertThat(recoveryStates.size(), equalTo(SHARD_COUNT));
        for (RecoveryState recoveryState : recoveryStates) {
            RecoverySource.SnapshotRecoverySource recoverySource = new RecoverySource.SnapshotRecoverySource(((RecoverySource.SnapshotRecoverySource) recoveryState.getRecoverySource()).restoreUUID(), new Snapshot(REPO_NAME, snapshotInfo.snapshotId()), Version.CURRENT, indexName);
            assertRecoveryState(recoveryState, 0, recoverySource, true, RecoveryState.Stage.DONE, null, nodeA);
            validateIndexRecoveryState(recoveryState.getIndex());
        }
    }
}

Example 22 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project crate by crate.

the class BlobStoreRepositoryRestoreTests method testRestoreSnapshotWithExistingFiles.

/**
 * Restoring a snapshot that contains multiple files must succeed even when
 * some files already exist in the shard's store.
 */
public void testRestoreSnapshotWithExistingFiles() throws IOException {
    final IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
    final ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 0);
    IndexShard shard = newShard(shardId, true);
    try {
        // index documents in the shards
        final int numDocs = scaledRandomIntBetween(1, 500);
        recoverShardFromStore(shard);
        for (int i = 0; i < numDocs; i++) {
            indexDoc(shard, Integer.toString(i));
            if (rarely()) {
                flushShard(shard, false);
            }
        }
        assertDocCount(shard, numDocs);
        // snapshot the shard
        final Repository repository = createRepository();
        final Snapshot snapshot = new Snapshot(repository.getMetadata().name(), new SnapshotId(randomAlphaOfLength(10), "_uuid"));
        snapshotShard(shard, snapshot, repository);
        // capture current store files
        final Store.MetadataSnapshot storeFiles = shard.snapshotStoreMetadata();
        assertFalse(storeFiles.asMap().isEmpty());
        // close the shard
        closeShards(shard);
        // delete some random files in the store
        List<String> deletedFiles = randomSubsetOf(randomIntBetween(1, storeFiles.size() - 1), storeFiles.asMap().keySet());
        for (String deletedFile : deletedFiles) {
            Files.delete(shard.shardPath().resolveIndex().resolve(deletedFile));
        }
        // build a new shard using the same store directory as the closed shard
        ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE);
        shard = newShard(shardRouting, shard.shardPath(), shard.indexSettings().getIndexMetadata(), null, new InternalEngineFactory(), () -> {
        }, RetentionLeaseSyncer.EMPTY, EMPTY_EVENT_LISTENER);
        // restore the shard
        recoverShardFromSnapshot(shard, snapshot, repository);
        // check that the shard is not corrupted
        TestUtil.checkIndex(shard.store().directory());
        // check that all files have been restored
        final Directory directory = shard.store().directory();
        final List<String> directoryFiles = Arrays.asList(directory.listAll());
        for (StoreFileMetadata storeFile : storeFiles) {
            String fileName = storeFile.name();
            assertTrue("File [" + fileName + "] does not exist in store directory", directoryFiles.contains(fileName));
            assertEquals(storeFile.length(), shard.store().directory().fileLength(fileName));
        }
    } finally {
        if (shard != null && shard.state() != IndexShardState.CLOSED) {
            try {
                shard.close("test", false);
            } finally {
                IOUtils.close(shard.store());
            }
        }
    }
}

Example 23 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project crate by crate.

the class BlobStoreRepositoryRestoreTests method testSnapshotWithConflictingName.

public void testSnapshotWithConflictingName() throws IOException {
    final IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
    final ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 0);
    IndexShard shard = newShard(shardId, true);
    try {
        // index documents in the shards
        final int numDocs = scaledRandomIntBetween(1, 500);
        recoverShardFromStore(shard);
        for (int i = 0; i < numDocs; i++) {
            indexDoc(shard, Integer.toString(i));
            if (rarely()) {
                flushShard(shard, false);
            }
        }
        assertDocCount(shard, numDocs);
        // snapshot the shard
        final Repository repository = createRepository();
        final Snapshot snapshot = new Snapshot(repository.getMetadata().name(), new SnapshotId(randomAlphaOfLength(10), "_uuid"));
        final String shardGen = snapshotShard(shard, snapshot, repository);
        assertNotNull(shardGen);
        final Snapshot snapshotWithSameName = new Snapshot(repository.getMetadata().name(), new SnapshotId(snapshot.getSnapshotId().getName(), "_uuid2"));
        final PlainActionFuture<SnapshotInfo> future = PlainActionFuture.newFuture();
        repository.finalizeSnapshot(snapshot.getSnapshotId(), ShardGenerations.builder().put(indexId, 0, shardGen).build(), 0L, null, 1, Collections.emptyList(), -1L, false, Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), true, future);
        future.actionGet();
        IndexShardSnapshotFailedException isfe = expectThrows(IndexShardSnapshotFailedException.class, () -> snapshotShard(shard, snapshotWithSameName, repository));
        assertThat(isfe.getMessage(), containsString("Duplicate snapshot name"));
    } finally {
        if (shard != null && shard.state() != IndexShardState.CLOSED) {
            try {
                shard.close("test", false);
            } finally {
                IOUtils.close(shard.store());
            }
        }
    }
}

Example 24 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project crate by crate.

the class SnapshotRestoreDDLDispatcherTest method testResolveMultiTablesIndexNamesFromSnapshot.

@Test
public void testResolveMultiTablesIndexNamesFromSnapshot() throws Exception {
    List<RestoreSnapshotAnalyzedStatement.RestoreTableInfo> tables = Arrays.asList(new RestoreSnapshotAnalyzedStatement.RestoreTableInfo(new TableIdent(null, "my_table"), null), new RestoreSnapshotAnalyzedStatement.RestoreTableInfo(new TableIdent(null, "my_partitioned_table"), null));
    List<SnapshotInfo> snapshots = Arrays.asList(new SnapshotInfo(new Snapshot("snapshot01", Collections.singletonList(".partitioned.my_partitioned_table.046jcchm6krj4e1g60o30c0"), 0)), new SnapshotInfo(new Snapshot("snapshot03", Collections.singletonList("my_table"), 0)));
    CompletableFuture<List<String>> future = new CompletableFuture<>();
    SnapshotRestoreDDLDispatcher.ResolveFromSnapshotActionListener actionListener = new SnapshotRestoreDDLDispatcher.ResolveFromSnapshotActionListener(future, tables, new HashSet<>());
    // need to mock here as constructor is not accessible
    GetSnapshotsResponse response = mock(GetSnapshotsResponse.class);
    when(response.getSnapshots()).thenReturn(snapshots);
    actionListener.onResponse(response);
    assertThat(future.get(), containsInAnyOrder("my_table", PartitionName.templateName(null, "my_partitioned_table") + "*"));
}

Example 25 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project elasticsearch by elastic.

the class TransportRestoreSnapshotAction method masterOperation.

@Override
protected void masterOperation(final RestoreSnapshotRequest request, final ClusterState state, final ActionListener<RestoreSnapshotResponse> listener) {
    RestoreService.RestoreRequest restoreRequest = new RestoreService.RestoreRequest(request.repository(), request.snapshot(), request.indices(), request.indicesOptions(), request.renamePattern(), request.renameReplacement(), request.settings(), request.masterNodeTimeout(), request.includeGlobalState(), request.partial(), request.includeAliases(), request.indexSettings(), request.ignoreIndexSettings(), "restore_snapshot[" + request.snapshot() + "]");
    restoreService.restoreSnapshot(restoreRequest, new ActionListener<RestoreCompletionResponse>() {

        @Override
        public void onResponse(RestoreCompletionResponse restoreCompletionResponse) {
            if (restoreCompletionResponse.getRestoreInfo() == null && request.waitForCompletion()) {
                final Snapshot snapshot = restoreCompletionResponse.getSnapshot();
                ClusterStateListener clusterStateListener = new ClusterStateListener() {

                    @Override
                    public void clusterChanged(ClusterChangedEvent changedEvent) {
                        final RestoreInProgress.Entry prevEntry = restoreInProgress(changedEvent.previousState(), snapshot);
                        final RestoreInProgress.Entry newEntry = restoreInProgress(changedEvent.state(), snapshot);
                        if (prevEntry == null) {
                            // When there is a master failure after a restore has been started, this listener might not be registered
                            // on the current master and as such it might miss some intermediary cluster states due to batching.
                            // Clean up listener in that case and acknowledge completion of restore operation to client.
                            clusterService.removeListener(this);
                            listener.onResponse(new RestoreSnapshotResponse(null));
                        } else if (newEntry == null) {
                            clusterService.removeListener(this);
                            ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards = prevEntry.shards();
                            assert prevEntry.state().completed() : "expected completed snapshot state but was " + prevEntry.state();
                            assert RestoreService.completed(shards) : "expected all restore entries to be completed";
                            RestoreInfo ri = new RestoreInfo(prevEntry.snapshot().getSnapshotId().getName(), prevEntry.indices(), shards.size(), shards.size() - RestoreService.failedShards(shards));
                            RestoreSnapshotResponse response = new RestoreSnapshotResponse(ri);
                            logger.debug("restore of [{}] completed", snapshot);
                            listener.onResponse(response);
                        } else {
                        // restore not completed yet, wait for next cluster state update
                        }
                    }
                };
                clusterService.addListener(clusterStateListener);
            } else {
                listener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.getRestoreInfo()));
            }
        }

        @Override
        public void onFailure(Exception t) {
            listener.onFailure(t);
        }
    });
}