Examples with RecoveryTarget org.opensearch.indices.recovery.RecoveryTarget used on opensource projects

Example 1 with RecoveryTarget

use of org.opensearch.indices.recovery.RecoveryTarget in project OpenSearch by opensearch-project.

the class IndexLevelReplicationTests method testAppendWhileRecovering.

public void testAppendWhileRecovering() throws Exception {
    try (ReplicationGroup shards = createGroup(0)) {
        shards.startAll();
        CountDownLatch latch = new CountDownLatch(2);
        int numDocs = randomIntBetween(100, 200);
        // just append one to the translog so we can assert below
        shards.appendDocs(1);
        Thread thread = new Thread() {

            @Override
            public void run() {
                try {
                    latch.countDown();
                    latch.await();
                    shards.appendDocs(numDocs - 1);
                } catch (Exception e) {
                    throw new AssertionError(e);
                }
            }
        };
        thread.start();
        IndexShard replica = shards.addReplica();
        Future<Void> future = shards.asyncRecoverReplica(replica, (indexShard, node) -> new RecoveryTarget(indexShard, node, recoveryListener) {

            @Override
            public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetadata, ActionListener<Void> listener) {
                super.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetadata, ActionListener.runAfter(listener, () -> {
                    latch.countDown();
                    try {
                        latch.await();
                    } catch (InterruptedException e) {
                        throw new AssertionError(e);
                    }
                }));
            }
        });
        future.get();
        thread.join();
        shards.assertAllEqual(numDocs);
        Engine engine = IndexShardTests.getEngineFromShard(shards.getPrimary());
        assertEquals(0, InternalEngineTests.getNumIndexVersionsLookups((InternalEngine) engine));
        assertEquals(0, InternalEngineTests.getNumVersionLookups((InternalEngine) engine));
    }
}

Example 2 with RecoveryTarget

use of org.opensearch.indices.recovery.RecoveryTarget in project OpenSearch by opensearch-project.

the class RecoveryDuringReplicationTests method testDoNotWaitForPendingSeqNo.

public void testDoNotWaitForPendingSeqNo() throws Exception {
    IndexMetadata metadata = buildIndexMetadata(1);
    final int pendingDocs = randomIntBetween(1, 5);
    final BlockingEngineFactory primaryEngineFactory = new BlockingEngineFactory();
    try (ReplicationGroup shards = new ReplicationGroup(metadata) {

        @Override
        protected EngineFactory getEngineFactory(ShardRouting routing) {
            if (routing.primary()) {
                return primaryEngineFactory;
            } else {
                return new InternalEngineFactory();
            }
        }
    }) {
        shards.startAll();
        int docs = shards.indexDocs(randomIntBetween(1, 10));
        // simulate a background global checkpoint sync at which point we expect the global checkpoint to advance on the replicas
        shards.syncGlobalCheckpoint();
        IndexShard replica = shards.getReplicas().get(0);
        shards.removeReplica(replica);
        closeShards(replica);
        docs += pendingDocs;
        primaryEngineFactory.latchIndexers(pendingDocs);
        CountDownLatch pendingDocsDone = new CountDownLatch(pendingDocs);
        for (int i = 0; i < pendingDocs; i++) {
            final String id = "pending_" + i;
            threadPool.generic().submit(() -> {
                try {
                    shards.index(new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON));
                } catch (Exception e) {
                    throw new AssertionError(e);
                } finally {
                    pendingDocsDone.countDown();
                }
            });
        }
        // wait for the pending ops to "hang"
        primaryEngineFactory.awaitIndexersLatch();
        primaryEngineFactory.allowIndexing();
        // index some more
        docs += shards.indexDocs(randomInt(5));
        IndexShard newReplica = shards.addReplicaWithExistingPath(replica.shardPath(), replica.routingEntry().currentNodeId());
        CountDownLatch recoveryStart = new CountDownLatch(1);
        AtomicBoolean recoveryDone = new AtomicBoolean(false);
        final Future<Void> recoveryFuture = shards.asyncRecoverReplica(newReplica, (indexShard, node) -> {
            recoveryStart.countDown();
            return new RecoveryTarget(indexShard, node, recoveryListener) {

                @Override
                public void finalizeRecovery(long globalCheckpoint, long trimAboveSeqNo, ActionListener<Void> listener) {
                    recoveryDone.set(true);
                    super.finalizeRecovery(globalCheckpoint, trimAboveSeqNo, listener);
                }
            };
        });
        recoveryStart.await();
        // index some more
        final int indexedDuringRecovery = shards.indexDocs(randomInt(5));
        docs += indexedDuringRecovery;
        assertBusy(() -> assertTrue("recovery should not wait for on pending docs", recoveryDone.get()));
        primaryEngineFactory.releaseLatchedIndexers();
        pendingDocsDone.await();
        // now recovery can finish
        recoveryFuture.get();
        assertThat(newReplica.recoveryState().getIndex().fileDetails(), empty());
        shards.assertAllEqual(docs);
    } finally {
        primaryEngineFactory.close();
    }
}

Example 3 with RecoveryTarget

use of org.opensearch.indices.recovery.RecoveryTarget in project OpenSearch by opensearch-project.

the class IndexShardTests method testTranslogRecoverySyncsTranslog.

public void testTranslogRecoverySyncsTranslog() throws IOException {
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetadata metadata = IndexMetadata.builder("test").putMapping("_doc", "{ \"properties\": { \"foo\":  { \"type\": \"text\"}}}").settings(settings).primaryTerm(0, 1).build();
    IndexShard primary = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, null);
    recoverShardFromStore(primary);
    indexDoc(primary, "_doc", "0", "{\"foo\" : \"bar\"}");
    IndexShard replica = newShard(primary.shardId(), false, "n2", metadata, null);
    recoverReplica(replica, primary, (shard, discoveryNode) -> new RecoveryTarget(shard, discoveryNode, recoveryListener) {

        @Override
        public void indexTranslogOperations(final List<Translog.Operation> operations, final int totalTranslogOps, final long maxSeenAutoIdTimestamp, final long maxSeqNoOfUpdatesOrDeletes, final RetentionLeases retentionLeases, final long mappingVersion, final ActionListener<Long> listener) {
            super.indexTranslogOperations(operations, totalTranslogOps, maxSeenAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, mappingVersion, ActionListener.wrap(r -> {
                assertFalse(replica.isSyncNeeded());
                listener.onResponse(r);
            }, listener::onFailure));
        }
    }, true, true);
    closeShards(primary, replica);
}

Example 4 with RecoveryTarget

use of org.opensearch.indices.recovery.RecoveryTarget in project OpenSearch by opensearch-project.

the class IndexShardTestCase method recoverUnstartedReplica.

/**
 * Recovers a replica from the give primary, allow the user to supply a custom recovery target. A typical usage of a custom recovery
 * target is to assert things in the various stages of recovery.
 *
 * Note: this method keeps the shard in {@link IndexShardState#POST_RECOVERY} and doesn't start it.
 *
 * @param replica                the recovery target shard
 * @param primary                the recovery source shard
 * @param targetSupplier         supplies an instance of {@link RecoveryTarget}
 * @param markAsRecovering       set to {@code false} if the replica is marked as recovering
 */
protected final void recoverUnstartedReplica(final IndexShard replica, final IndexShard primary, final BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier, final boolean markAsRecovering, final Set<String> inSyncIds, final IndexShardRoutingTable routingTable) throws IOException {
    final DiscoveryNode pNode = getFakeDiscoNode(primary.routingEntry().currentNodeId());
    final DiscoveryNode rNode = getFakeDiscoNode(replica.routingEntry().currentNodeId());
    if (markAsRecovering) {
        replica.markAsRecovering("remote", new RecoveryState(replica.routingEntry(), pNode, rNode));
    } else {
        assertEquals(replica.state(), IndexShardState.RECOVERING);
    }
    replica.prepareForIndexRecovery();
    final RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
    final long startingSeqNo = recoveryTarget.indexShard().recoverLocallyUpToGlobalCheckpoint();
    final StartRecoveryRequest request = PeerRecoveryTargetService.getStartRecoveryRequest(logger, rNode, recoveryTarget, startingSeqNo);
    int fileChunkSizeInBytes = Math.toIntExact(randomBoolean() ? RecoverySettings.DEFAULT_CHUNK_SIZE.getBytes() : randomIntBetween(1, 10 * 1024 * 1024));
    final RecoverySourceHandler recovery = new RecoverySourceHandler(primary, new AsyncRecoveryTarget(recoveryTarget, threadPool.generic()), threadPool, request, fileChunkSizeInBytes, between(1, 8), between(1, 8));
    primary.updateShardState(primary.routingEntry(), primary.getPendingPrimaryTerm(), null, currentClusterStateVersion.incrementAndGet(), inSyncIds, routingTable);
    try {
        PlainActionFuture<RecoveryResponse> future = new PlainActionFuture<>();
        recovery.recoverToTarget(future);
        future.actionGet();
        recoveryTarget.markAsDone();
    } catch (Exception e) {
        recoveryTarget.fail(new RecoveryFailedException(request, e), false);
        throw e;
    }
}

Example 5 with RecoveryTarget

use of org.opensearch.indices.recovery.RecoveryTarget in project OpenSearch by opensearch-project.

the class IndexLevelReplicationTests method testAppendOnlyRecoveryThenReplication.

public void testAppendOnlyRecoveryThenReplication() throws Exception {
    CountDownLatch indexedOnPrimary = new CountDownLatch(1);
    CountDownLatch recoveryDone = new CountDownLatch(1);
    try (ReplicationGroup shards = new ReplicationGroup(buildIndexMetadata(1)) {

        @Override
        protected EngineFactory getEngineFactory(ShardRouting routing) {
            return config -> new InternalEngine(config) {

                @Override
                public IndexResult index(Index op) throws IOException {
                    IndexResult result = super.index(op);
                    if (op.origin() == Operation.Origin.PRIMARY) {
                        indexedOnPrimary.countDown();
                        // to make sure that this operation is replicated to the replica via recovery, then via replication.
                        try {
                            recoveryDone.await();
                        } catch (InterruptedException e) {
                            throw new AssertionError(e);
                        }
                    }
                    return result;
                }
            };
        }
    }) {
        shards.startAll();
        Thread thread = new Thread(() -> {
            IndexRequest indexRequest = new IndexRequest(index.getName()).source("{}", XContentType.JSON);
            try {
                shards.index(indexRequest);
            } catch (Exception e) {
                throw new AssertionError(e);
            }
        });
        thread.start();
        IndexShard replica = shards.addReplica();
        Future<Void> fut = shards.asyncRecoverReplica(replica, (shard, node) -> new RecoveryTarget(shard, node, recoveryListener) {

            @Override
            public void prepareForTranslogOperations(int totalTranslogOps, ActionListener<Void> listener) {
                try {
                    indexedOnPrimary.await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                }
                super.prepareForTranslogOperations(totalTranslogOps, listener);
            }
        });
        fut.get();
        recoveryDone.countDown();
        thread.join();
        shards.assertAllEqual(1);
    }
}