Examples with Snapshot org.elasticsearch.snapshots.Snapshot used on opensource projects

Example 31 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project elasticsearch by elastic.

the class ClusterSerializationTests method testSnapshotDeletionsInProgressSerialization.

public void testSnapshotDeletionsInProgressSerialization() throws Exception {
    boolean includeRestore = randomBoolean();
    ClusterState.Builder builder = ClusterState.builder(ClusterState.EMPTY_STATE).putCustom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.newInstance(new SnapshotDeletionsInProgress.Entry(new Snapshot("repo1", new SnapshotId("snap1", UUIDs.randomBase64UUID())), randomNonNegativeLong(), randomNonNegativeLong())));
    if (includeRestore) {
        builder.putCustom(RestoreInProgress.TYPE, new RestoreInProgress(new RestoreInProgress.Entry(new Snapshot("repo2", new SnapshotId("snap2", UUIDs.randomBase64UUID())), RestoreInProgress.State.STARTED, Collections.singletonList("index_name"), ImmutableOpenMap.of())));
    }
    ClusterState clusterState = builder.incrementVersion().build();
    Diff<ClusterState> diffs = clusterState.diff(ClusterState.EMPTY_STATE);
    // serialize with current version
    BytesStreamOutput outStream = new BytesStreamOutput();
    diffs.writeTo(outStream);
    StreamInput inStream = outStream.bytes().streamInput();
    inStream = new NamedWriteableAwareStreamInput(inStream, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
    Diff<ClusterState> serializedDiffs = ClusterState.readDiffFrom(inStream, clusterState.nodes().getLocalNode());
    ClusterState stateAfterDiffs = serializedDiffs.apply(ClusterState.EMPTY_STATE);
    assertThat(stateAfterDiffs.custom(RestoreInProgress.TYPE), includeRestore ? notNullValue() : nullValue());
    assertThat(stateAfterDiffs.custom(SnapshotDeletionsInProgress.TYPE), notNullValue());
    // serialize with old version
    outStream = new BytesStreamOutput();
    outStream.setVersion(Version.CURRENT.minimumCompatibilityVersion());
    diffs.writeTo(outStream);
    inStream = outStream.bytes().streamInput();
    inStream = new NamedWriteableAwareStreamInput(inStream, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
    serializedDiffs = ClusterState.readDiffFrom(inStream, clusterState.nodes().getLocalNode());
    stateAfterDiffs = serializedDiffs.apply(ClusterState.EMPTY_STATE);
    assertThat(stateAfterDiffs.custom(RestoreInProgress.TYPE), includeRestore ? notNullValue() : nullValue());
    assertThat(stateAfterDiffs.custom(SnapshotDeletionsInProgress.TYPE), nullValue());
    // remove the custom and try serializing again with old version
    clusterState = ClusterState.builder(clusterState).removeCustom(SnapshotDeletionsInProgress.TYPE).incrementVersion().build();
    outStream = new BytesStreamOutput();
    diffs.writeTo(outStream);
    inStream = outStream.bytes().streamInput();
    inStream = new NamedWriteableAwareStreamInput(inStream, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
    serializedDiffs = ClusterState.readDiffFrom(inStream, clusterState.nodes().getLocalNode());
    stateAfterDiffs = serializedDiffs.apply(stateAfterDiffs);
    assertThat(stateAfterDiffs.custom(RestoreInProgress.TYPE), includeRestore ? notNullValue() : nullValue());
    assertThat(stateAfterDiffs.custom(SnapshotDeletionsInProgress.TYPE), nullValue());
}

Example 32 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project elasticsearch by elastic.

the class ExceptionSerializationTests method testSnapshotException.

public void testSnapshotException() throws IOException {
    final Snapshot snapshot = new Snapshot("repo", new SnapshotId("snap", UUIDs.randomBase64UUID()));
    SnapshotException ex = serialize(new SnapshotException(snapshot, "no such snapshot", new NullPointerException()));
    assertEquals(ex.getRepositoryName(), snapshot.getRepository());
    assertEquals(ex.getSnapshotName(), snapshot.getSnapshotId().getName());
    assertEquals(ex.getMessage(), "[" + snapshot + "] no such snapshot");
    assertTrue(ex.getCause() instanceof NullPointerException);
    ex = serialize(new SnapshotException(null, "no such snapshot", new NullPointerException()));
    assertNull(ex.getRepositoryName());
    assertNull(ex.getSnapshotName());
    assertEquals(ex.getMessage(), "[_na] no such snapshot");
    assertTrue(ex.getCause() instanceof NullPointerException);
}

Example 33 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project elasticsearch by elastic.

the class MetaDataDeleteIndexServiceTests method testDeleteSnapshotting.

public void testDeleteSnapshotting() {
    String index = randomAsciiOfLength(5);
    Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid"));
    SnapshotsInProgress snaps = new SnapshotsInProgress(new SnapshotsInProgress.Entry(snapshot, true, false, SnapshotsInProgress.State.INIT, singletonList(new IndexId(index, "doesn't matter")), System.currentTimeMillis(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of()));
    ClusterState state = ClusterState.builder(clusterState(index)).putCustom(SnapshotsInProgress.TYPE, snaps).build();
    Exception e = expectThrows(IllegalArgumentException.class, () -> service.deleteIndices(state, singleton(state.metaData().getIndices().get(index).getIndex())));
    assertEquals("Cannot delete indices that are being snapshotted: [[" + index + "]]. Try again after snapshot finishes " + "or cancel the currently running snapshot.", e.getMessage());
}

Example 34 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project crate by crate.

the class SnapshotRestoreIntegrationTest method waitForCompletion.

private SnapshotInfo waitForCompletion(String repository, String snapshotName, TimeValue timeout) throws InterruptedException {
    long start = System.currentTimeMillis();
    Snapshot snapshot = new Snapshot(repository, new SnapshotId(repository, snapshotName));
    while (System.currentTimeMillis() - start < timeout.millis()) {
        List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots(repository).setSnapshots(snapshotName).get().getSnapshots();
        assertThat(snapshotInfos.size(), equalTo(1));
        if (snapshotInfos.get(0).state().completed()) {
            // Make sure that snapshot clean up operations are finished
            ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get();
            SnapshotsInProgress snapshotsInProgress = stateResponse.getState().custom(SnapshotsInProgress.TYPE);
            if (snapshotsInProgress == null || snapshotsInProgress.snapshot(snapshot) == null) {
                return snapshotInfos.get(0);
            }
        }
        Thread.sleep(100);
    }
    fail("Timeout waiting for snapshot completion!");
    return null;
}

Example 35 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project crate by crate.

the class BlobStoreRepository method restoreShard.

@Override
public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState, ActionListener<Void> listener) {
    final ShardId shardId = store.shardId();
    final ActionListener<Void> restoreListener = ActionListener.delegateResponse(listener, (l, e) -> l.onFailure(new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId + "]", e)));
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    final BlobContainer container = shardContainer(indexId, snapshotShardId);
    executor.execute(ActionRunnable.wrap(restoreListener, l -> {
        final BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(container, snapshotId);
        final SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
        new FileRestoreContext(metadata.name(), shardId, snapshotId, recoveryState) {

            @Override
            protected void restoreFiles(List<BlobStoreIndexShardSnapshot.FileInfo> filesToRecover, Store store, ActionListener<Void> listener) {
                if (filesToRecover.isEmpty()) {
                    listener.onResponse(null);
                } else {
                    // Start as many workers as fit into the snapshot pool at once at the most
                    int maxPoolSize = executor instanceof ThreadPoolExecutor ? ((ThreadPoolExecutor) executor).getMaximumPoolSize() : 1;
                    final int workers = Math.min(maxPoolSize, snapshotFiles.indexFiles().size());
                    final BlockingQueue<BlobStoreIndexShardSnapshot.FileInfo> files = new LinkedBlockingQueue<>(filesToRecover);
                    final ActionListener<Void> allFilesListener = fileQueueListener(files, workers, ActionListener.map(listener, v -> null));
                    // restore the files from the snapshot to the Lucene store
                    for (int i = 0; i < workers; ++i) {
                        executor.execute(ActionRunnable.run(allFilesListener, () -> {
                            store.incRef();
                            try {
                                BlobStoreIndexShardSnapshot.FileInfo fileToRecover;
                                while ((fileToRecover = files.poll(0L, TimeUnit.MILLISECONDS)) != null) {
                                    restoreFile(fileToRecover, store);
                                }
                            } finally {
                                store.decRef();
                            }
                        }));
                    }
                }
            }

            private void restoreFile(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Store store) throws IOException {
                boolean success = false;
                try (InputStream stream = maybeRateLimit(new SlicedInputStream(fileInfo.numberOfParts()) {

                    @Override
                    protected InputStream openSlice(long slice) throws IOException {
                        return container.readBlob(fileInfo.partName(slice));
                    }
                }, restoreRateLimiter, restoreRateLimitingTimeInNanos)) {
                    try (IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
                        final byte[] buffer = new byte[BUFFER_SIZE];
                        int length;
                        while ((length = stream.read(buffer)) > 0) {
                            indexOutput.writeBytes(buffer, 0, length);
                            recoveryState.getIndex().addRecoveredBytesToFile(fileInfo.physicalName(), length);
                        }
                        Store.verify(indexOutput);
                        indexOutput.close();
                        store.directory().sync(Collections.singleton(fileInfo.physicalName()));
                        success = true;
                    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
                        try {
                            store.markStoreCorrupted(ex);
                        } catch (IOException e) {
                            LOGGER.warn("store cannot be marked as corrupted", e);
                        }
                        throw ex;
                    } finally {
                        if (success == false) {
                            store.deleteQuiet(fileInfo.physicalName());
                        }
                    }
                }
            }
        }.restore(snapshotFiles, store, l);
    }));
}