Examples with Snapshot org.opensearch.snapshots.Snapshot used on opensource projects

Example 11 with Snapshot

use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.

the class BlobStoreRepositoryRestoreTests method testSnapshotWithConflictingName.

public void testSnapshotWithConflictingName() throws Exception {
    final IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
    final ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 0);
    IndexShard shard = newShard(shardId, true);
    try {
        // index documents in the shards
        final int numDocs = scaledRandomIntBetween(1, 500);
        recoverShardFromStore(shard);
        for (int i = 0; i < numDocs; i++) {
            indexDoc(shard, "_doc", Integer.toString(i));
            if (rarely()) {
                flushShard(shard, false);
            }
        }
        assertDocCount(shard, numDocs);
        // snapshot the shard
        final Repository repository = createRepository();
        final Snapshot snapshot = new Snapshot(repository.getMetadata().name(), new SnapshotId(randomAlphaOfLength(10), "_uuid"));
        final String shardGen = snapshotShard(shard, snapshot, repository);
        assertNotNull(shardGen);
        final Snapshot snapshotWithSameName = new Snapshot(repository.getMetadata().name(), new SnapshotId(snapshot.getSnapshotId().getName(), "_uuid2"));
        final ShardGenerations shardGenerations = ShardGenerations.builder().put(indexId, 0, shardGen).build();
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(shardGenerations, RepositoryData.EMPTY_REPO_GEN, Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
        IndexShardSnapshotFailedException isfe = expectThrows(IndexShardSnapshotFailedException.class, () -> snapshotShard(shard, snapshotWithSameName, repository));
        assertThat(isfe.getMessage(), containsString("Duplicate snapshot name"));
    } finally {
        if (shard != null && shard.state() != IndexShardState.CLOSED) {
            try {
                shard.close("test", false);
            } finally {
                IOUtils.close(shard.store());
            }
        }
    }
}

Example 12 with Snapshot

use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.

the class IndexShardTests method testSnapshotWhileResettingEngine.

/**
 * This test simulates a scenario seen rarely in ConcurrentSeqNoVersioningIT. While engine is inside
 * resetEngineToGlobalCheckpoint snapshot metadata could fail
 */
public void testSnapshotWhileResettingEngine() throws Exception {
    CountDownLatch readyToSnapshotLatch = new CountDownLatch(1);
    CountDownLatch snapshotDoneLatch = new CountDownLatch(1);
    IndexShard shard = newStartedShard(false, Settings.EMPTY, config -> new InternalEngine(config) {

        @Override
        public InternalEngine recoverFromTranslog(TranslogRecoveryRunner translogRecoveryRunner, long recoverUpToSeqNo) throws IOException {
            InternalEngine engine = super.recoverFromTranslog(translogRecoveryRunner, recoverUpToSeqNo);
            readyToSnapshotLatch.countDown();
            try {
                snapshotDoneLatch.await();
            } catch (InterruptedException e) {
                throw new AssertionError(e);
            }
            return engine;
        }
    });
    indexOnReplicaWithGaps(shard, between(0, 1000), Math.toIntExact(shard.getLocalCheckpoint()));
    final long globalCheckpoint = randomLongBetween(shard.getLastKnownGlobalCheckpoint(), shard.getLocalCheckpoint());
    shard.updateGlobalCheckpointOnReplica(globalCheckpoint, "test");
    Thread snapshotThread = new Thread(() -> {
        try {
            readyToSnapshotLatch.await();
            shard.snapshotStoreMetadata();
            try (GatedCloseable<IndexCommit> wrappedIndexCommit = shard.acquireLastIndexCommit(randomBoolean())) {
                shard.store().getMetadata(wrappedIndexCommit.get());
            }
            try (GatedCloseable<IndexCommit> wrappedSafeCommit = shard.acquireSafeIndexCommit()) {
                shard.store().getMetadata(wrappedSafeCommit.get());
            }
        } catch (InterruptedException | IOException e) {
            throw new AssertionError(e);
        } finally {
            snapshotDoneLatch.countDown();
        }
    });
    snapshotThread.start();
    final CountDownLatch engineResetLatch = new CountDownLatch(1);
    shard.acquireAllReplicaOperationsPermits(shard.getOperationPrimaryTerm(), shard.getLastKnownGlobalCheckpoint(), 0L, ActionListener.wrap(r -> {
        try (Releasable dummy = r) {
            shard.resetEngineToGlobalCheckpoint();
        } finally {
            engineResetLatch.countDown();
        }
    }, Assert::assertNotNull), TimeValue.timeValueMinutes(1L));
    engineResetLatch.await();
    snapshotThread.join();
    closeShard(shard, false);
}

Example 13 with Snapshot

use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.

the class IndexShardTests method testResetEngineWithBrokenTranslog.

public void testResetEngineWithBrokenTranslog() throws Exception {
    IndexShard shard = newStartedShard(false);
    updateMappings(shard, IndexMetadata.builder(shard.indexSettings.getIndexMetadata()).putMapping("{ \"properties\": { \"foo\":  { \"type\": \"text\"}}}").build());
    final List<Translog.Operation> operations = Stream.concat(IntStream.range(0, randomIntBetween(0, 10)).mapToObj(n -> new Translog.Index("1", 0, shard.getPendingPrimaryTerm(), 1, "{\"foo\" : \"bar\"}".getBytes(Charset.forName("UTF-8")), null, -1)), // entries with corrupted source
    IntStream.range(0, randomIntBetween(1, 10)).mapToObj(n -> new Translog.Index("1", 0, shard.getPendingPrimaryTerm(), 1, "{\"foo\" : \"bar}".getBytes(Charset.forName("UTF-8")), null, -1))).collect(Collectors.toList());
    Randomness.shuffle(operations);
    final CountDownLatch engineResetLatch = new CountDownLatch(1);
    shard.acquireAllReplicaOperationsPermits(shard.getOperationPrimaryTerm(), shard.getLastKnownGlobalCheckpoint(), 0L, ActionListener.wrap(r -> {
        try (Releasable ignored = r) {
            Translog.Snapshot snapshot = TestTranslog.newSnapshotFromOperations(operations);
            final MapperParsingException error = expectThrows(MapperParsingException.class, () -> shard.runTranslogRecovery(shard.getEngine(), snapshot, Engine.Operation.Origin.LOCAL_RESET, () -> {
            }));
            assertThat(error.getMessage(), containsString("failed to parse field [foo] of type [text]"));
        } finally {
            engineResetLatch.countDown();
        }
    }, e -> {
        throw new AssertionError(e);
    }), TimeValue.timeValueMinutes(1));
    engineResetLatch.await();
    closeShards(shard);
}

Example 14 with Snapshot

use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.

the class IndexShardTests method testRestoreShard.

public void testRestoreShard() throws IOException {
    final IndexShard source = newStartedShard(true);
    IndexShard target = newStartedShard(true);
    indexDoc(source, "_doc", "0");
    // create a gap in the history
    EngineTestCase.generateNewSeqNo(source.getEngine());
    indexDoc(source, "_doc", "2");
    if (randomBoolean()) {
        source.refresh("test");
    }
    indexDoc(target, "_doc", "1");
    target.refresh("test");
    assertDocs(target, "1");
    // only flush source
    flushShard(source);
    ShardRouting routing = ShardRoutingHelper.initWithSameId(target.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE);
    final Snapshot snapshot = new Snapshot("foo", new SnapshotId("bar", UUIDs.randomBase64UUID()));
    routing = ShardRoutingHelper.newWithRestoreSource(routing, new RecoverySource.SnapshotRecoverySource(UUIDs.randomBase64UUID(), snapshot, Version.CURRENT, new IndexId("test", UUIDs.randomBase64UUID(random()))));
    target = reinitShard(target, routing);
    Store sourceStore = source.store();
    Store targetStore = target.store();
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    target.markAsRecovering("store", new RecoveryState(routing, localNode, null));
    final PlainActionFuture<Boolean> future = PlainActionFuture.newFuture();
    target.restoreFromRepository(new RestoreOnlyRepository("test") {

        @Override
        public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState, ActionListener<Void> listener) {
            ActionListener.completeWith(listener, () -> {
                cleanLuceneIndex(targetStore.directory());
                for (String file : sourceStore.directory().listAll()) {
                    if (file.equals("write.lock") || file.startsWith("extra")) {
                        continue;
                    }
                    targetStore.directory().copyFrom(sourceStore.directory(), file, file, IOContext.DEFAULT);
                }
                recoveryState.getIndex().setFileDetailsComplete();
                return null;
            });
        }
    }, future);
    assertTrue(future.actionGet());
    assertThat(target.getLocalCheckpoint(), equalTo(2L));
    assertThat(target.seqNoStats().getMaxSeqNo(), equalTo(2L));
    assertThat(target.seqNoStats().getGlobalCheckpoint(), equalTo(0L));
    IndexShardTestCase.updateRoutingEntry(target, routing.moveToStarted());
    assertThat(target.getReplicationTracker().getTrackedLocalCheckpointForShard(target.routingEntry().allocationId().getId()).getLocalCheckpoint(), equalTo(2L));
    assertThat(target.seqNoStats().getGlobalCheckpoint(), equalTo(2L));
    assertDocs(target, "0", "2");
    closeShard(source, false);
    closeShards(target);
}

Example 15 with Snapshot

use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.

the class MetadataDeleteIndexServiceTests method testDeleteSnapshotting.

public void testDeleteSnapshotting() {
    String index = randomAlphaOfLength(5);
    Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid"));
    SnapshotsInProgress snaps = SnapshotsInProgress.of(org.opensearch.common.collect.List.of(new SnapshotsInProgress.Entry(snapshot, true, false, SnapshotsInProgress.State.INIT, singletonList(new IndexId(index, "doesn't matter")), Collections.emptyList(), System.currentTimeMillis(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of(), null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()))));
    ClusterState state = ClusterState.builder(clusterState(index)).putCustom(SnapshotsInProgress.TYPE, snaps).build();
    Exception e = expectThrows(SnapshotInProgressException.class, () -> service.deleteIndices(state, singleton(state.metadata().getIndices().get(index).getIndex())));
    assertEquals("Cannot delete indices that are being snapshotted: [[" + index + "]]. Try again after snapshot finishes " + "or cancel the currently running snapshot.", e.getMessage());
}