Examples with RetentionLeases org.opensearch.index.seqno.RetentionLeases used on opensource projects

Example 21 with RetentionLeases

use of org.opensearch.index.seqno.RetentionLeases in project OpenSearch by opensearch-project.

the class IndexShardTests method testShardActiveDuringPeerRecovery.

public void testShardActiveDuringPeerRecovery() throws IOException {
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetadata metadata = IndexMetadata.builder("test").putMapping("{ \"properties\": { \"foo\":  { \"type\": \"text\"}}}").settings(settings).primaryTerm(0, 1).build();
    IndexShard primary = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, null);
    recoverShardFromStore(primary);
    indexDoc(primary, "_doc", "0", "{\"foo\" : \"bar\"}");
    IndexShard replica = newShard(primary.shardId(), false, "n2", metadata, null);
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    replica.markAsRecovering("for testing", new RecoveryState(replica.routingEntry(), localNode, localNode));
    // Shard is still inactive since we haven't started recovering yet
    assertFalse(replica.isActive());
    recoverReplica(replica, primary, (shard, discoveryNode) -> new RecoveryTarget(shard, discoveryNode, recoveryListener) {

        @Override
        public void indexTranslogOperations(final List<Translog.Operation> operations, final int totalTranslogOps, final long maxAutoIdTimestamp, final long maxSeqNoOfUpdatesOrDeletes, final RetentionLeases retentionLeases, final long mappingVersion, final ActionListener<Long> listener) {
            super.indexTranslogOperations(operations, totalTranslogOps, maxAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, mappingVersion, ActionListener.wrap(checkpoint -> {
                listener.onResponse(checkpoint);
                // Shard should now be active since we did recover:
                assertTrue(replica.isActive());
            }, listener::onFailure));
        }
    }, false, true);
    closeShards(primary, replica);
}

Example 22 with RetentionLeases

use of org.opensearch.index.seqno.RetentionLeases in project OpenSearch by opensearch-project.

the class IndexShardTests method testRefreshListenersDuringPeerRecovery.

public void testRefreshListenersDuringPeerRecovery() throws IOException {
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetadata metadata = IndexMetadata.builder("test").putMapping("{ \"properties\": { \"foo\":  { \"type\": \"text\"}}}").settings(settings).primaryTerm(0, 1).build();
    IndexShard primary = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, null);
    recoverShardFromStore(primary);
    indexDoc(primary, "_doc", "0", "{\"foo\" : \"bar\"}");
    Consumer<IndexShard> assertListenerCalled = shard -> {
        AtomicBoolean called = new AtomicBoolean();
        shard.addRefreshListener(null, b -> {
            assertFalse(b);
            called.set(true);
        });
        assertTrue(called.get());
    };
    IndexShard replica = newShard(primary.shardId(), false, "n2", metadata, null);
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    replica.markAsRecovering("for testing", new RecoveryState(replica.routingEntry(), localNode, localNode));
    assertListenerCalled.accept(replica);
    recoverReplica(replica, primary, (shard, discoveryNode) -> new RecoveryTarget(shard, discoveryNode, recoveryListener) {

        // we're only checking that listeners are called when the engine is open, before there is no point
        @Override
        public void prepareForTranslogOperations(int totalTranslogOps, ActionListener<Void> listener) {
            super.prepareForTranslogOperations(totalTranslogOps, ActionListener.wrap(r -> {
                assertListenerCalled.accept(replica);
                listener.onResponse(r);
            }, listener::onFailure));
        }

        @Override
        public void indexTranslogOperations(final List<Translog.Operation> operations, final int totalTranslogOps, final long maxAutoIdTimestamp, final long maxSeqNoOfUpdatesOrDeletes, final RetentionLeases retentionLeases, final long mappingVersion, final ActionListener<Long> listener) {
            super.indexTranslogOperations(operations, totalTranslogOps, maxAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, mappingVersion, ActionListener.wrap(r -> {
                assertListenerCalled.accept(replica);
                listener.onResponse(r);
            }, listener::onFailure));
        }

        @Override
        public void finalizeRecovery(long globalCheckpoint, long trimAboveSeqNo, ActionListener<Void> listener) {
            super.finalizeRecovery(globalCheckpoint, trimAboveSeqNo, ActionListener.wrap(r -> {
                assertListenerCalled.accept(replica);
                listener.onResponse(r);
            }, listener::onFailure));
        }
    }, false, true);
    closeShards(primary, replica);
}

Example 23 with RetentionLeases

use of org.opensearch.index.seqno.RetentionLeases in project OpenSearch by opensearch-project.

the class RecoveryDuringReplicationTests method testCheckpointsAndMarkingInSync.

public void testCheckpointsAndMarkingInSync() throws Exception {
    final IndexMetadata metadata = buildIndexMetadata(0);
    final BlockingEngineFactory replicaEngineFactory = new BlockingEngineFactory();
    try (ReplicationGroup shards = new ReplicationGroup(metadata) {

        @Override
        protected EngineFactory getEngineFactory(final ShardRouting routing) {
            if (routing.primary()) {
                return new InternalEngineFactory();
            } else {
                return replicaEngineFactory;
            }
        }
    };
        // make sure we release indexers before closing
        AutoCloseable ignored = replicaEngineFactory) {
        shards.startPrimary();
        final int docs = shards.indexDocs(randomIntBetween(1, 10));
        logger.info("indexed [{}] docs", docs);
        final CountDownLatch pendingDocDone = new CountDownLatch(1);
        final CountDownLatch pendingDocActiveWithExtraDocIndexed = new CountDownLatch(1);
        final CountDownLatch phaseTwoStartLatch = new CountDownLatch(1);
        final IndexShard replica = shards.addReplica();
        final Future<Void> recoveryFuture = shards.asyncRecoverReplica(replica, (indexShard, node) -> new RecoveryTarget(indexShard, node, recoveryListener) {

            @Override
            public void indexTranslogOperations(final List<Translog.Operation> operations, final int totalTranslogOps, final long maxAutoIdTimestamp, final long maxSeqNoOfUpdates, final RetentionLeases retentionLeases, final long mappingVersion, final ActionListener<Long> listener) {
                // index a doc which is not part of the snapshot, but also does not complete on replica
                replicaEngineFactory.latchIndexers(1);
                threadPool.generic().submit(() -> {
                    try {
                        shards.index(new IndexRequest(index.getName()).id("pending").source("{}", XContentType.JSON));
                    } catch (final Exception e) {
                        throw new RuntimeException(e);
                    } finally {
                        pendingDocDone.countDown();
                    }
                });
                try {
                    // the pending doc is latched in the engine
                    replicaEngineFactory.awaitIndexersLatch();
                    // unblock indexing for the next doc
                    replicaEngineFactory.allowIndexing();
                    shards.index(new IndexRequest(index.getName()).id("completed").source("{}", XContentType.JSON));
                    pendingDocActiveWithExtraDocIndexed.countDown();
                } catch (final Exception e) {
                    throw new AssertionError(e);
                }
                try {
                    phaseTwoStartLatch.await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                }
                super.indexTranslogOperations(operations, totalTranslogOps, maxAutoIdTimestamp, maxSeqNoOfUpdates, retentionLeases, mappingVersion, listener);
            }
        });
        pendingDocActiveWithExtraDocIndexed.await();
        assertThat(pendingDocDone.getCount(), equalTo(1L));
        {
            final long expectedDocs = docs + 2L;
            assertThat(shards.getPrimary().getLocalCheckpoint(), equalTo(expectedDocs - 1));
            // recovery has not completed, therefore the global checkpoint can have advanced on the primary
            assertThat(shards.getPrimary().getLastKnownGlobalCheckpoint(), equalTo(expectedDocs - 1));
            // the pending document is not done, the checkpoints can not have advanced on the replica
            assertThat(replica.getLocalCheckpoint(), lessThan(expectedDocs - 1));
            assertThat(replica.getLastKnownGlobalCheckpoint(), lessThan(expectedDocs - 1));
        }
        // wait for recovery to enter the translog phase
        phaseTwoStartLatch.countDown();
        // wait for the translog phase to complete and the recovery to block global checkpoint advancement
        assertBusy(() -> assertTrue(shards.getPrimary().pendingInSync()));
        {
            shards.index(new IndexRequest(index.getName()).id("last").source("{}", XContentType.JSON));
            final long expectedDocs = docs + 3L;
            assertThat(shards.getPrimary().getLocalCheckpoint(), equalTo(expectedDocs - 1));
            // recovery is now in the process of being completed, therefore the global checkpoint can not have advanced on the primary
            assertThat(shards.getPrimary().getLastKnownGlobalCheckpoint(), equalTo(expectedDocs - 2));
            assertThat(replica.getLocalCheckpoint(), lessThan(expectedDocs - 2));
            assertThat(replica.getLastKnownGlobalCheckpoint(), lessThan(expectedDocs - 2));
        }
        replicaEngineFactory.releaseLatchedIndexers();
        pendingDocDone.await();
        recoveryFuture.get();
        {
            final long expectedDocs = docs + 3L;
            assertBusy(() -> {
                assertThat(shards.getPrimary().getLocalCheckpoint(), equalTo(expectedDocs - 1));
                assertThat(shards.getPrimary().getLastKnownGlobalCheckpoint(), equalTo(expectedDocs - 1));
                assertThat(replica.getLocalCheckpoint(), equalTo(expectedDocs - 1));
                // the global checkpoint advances can only advance here if a background global checkpoint sync fires
                assertThat(replica.getLastKnownGlobalCheckpoint(), anyOf(equalTo(expectedDocs - 1), equalTo(expectedDocs - 2)));
            });
        }
    }
}

Example 24 with RetentionLeases

use of org.opensearch.index.seqno.RetentionLeases in project OpenSearch by opensearch-project.

the class RetentionLeasesReplicationTests method testSimpleSyncRetentionLeases.

public void testSimpleSyncRetentionLeases() throws Exception {
    Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build();
    try (ReplicationGroup group = createGroup(between(0, 2), settings)) {
        group.startAll();
        List<RetentionLease> leases = new ArrayList<>();
        int iterations = between(1, 100);
        CountDownLatch latch = new CountDownLatch(iterations);
        for (int i = 0; i < iterations; i++) {
            if (leases.isEmpty() == false && rarely()) {
                RetentionLease leaseToRemove = randomFrom(leases);
                leases.remove(leaseToRemove);
                group.removeRetentionLease(leaseToRemove.id(), ActionListener.wrap(latch::countDown));
            } else {
                RetentionLease newLease = group.addRetentionLease(Integer.toString(i), randomNonNegativeLong(), "test-" + i, ActionListener.wrap(latch::countDown));
                leases.add(newLease);
            }
        }
        RetentionLeases leasesOnPrimary = group.getPrimary().getRetentionLeases();
        assertThat(leasesOnPrimary.version(), equalTo(iterations + group.getReplicas().size() + 1L));
        assertThat(leasesOnPrimary.primaryTerm(), equalTo(group.getPrimary().getOperationPrimaryTerm()));
        assertThat(RetentionLeaseUtils.toMapExcludingPeerRecoveryRetentionLeases(leasesOnPrimary).values(), containsInAnyOrder(leases.toArray(new RetentionLease[0])));
        latch.await();
        for (IndexShard replica : group.getReplicas()) {
            assertThat(replica.getRetentionLeases(), equalTo(leasesOnPrimary));
        }
    }
}

Example 25 with RetentionLeases

use of org.opensearch.index.seqno.RetentionLeases in project OpenSearch by opensearch-project.

the class RetentionLeasesReplicationTests method testSyncRetentionLeasesWithPrimaryPromotion.

public void testSyncRetentionLeasesWithPrimaryPromotion() throws Exception {
    Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build();
    int numberOfReplicas = between(2, 4);
    IndexMetadata indexMetadata = buildIndexMetadata(numberOfReplicas, settings, indexMapping);
    try (ReplicationGroup group = new ReplicationGroup(indexMetadata) {

        @Override
        protected void syncRetentionLeases(ShardId shardId, RetentionLeases leases, ActionListener<ReplicationResponse> listener) {
            listener.onResponse(new SyncRetentionLeasesResponse(new RetentionLeaseSyncAction.Request(shardId, leases)));
        }
    }) {
        group.startAll();
        for (IndexShard replica : group.getReplicas()) {
            replica.updateRetentionLeasesOnReplica(group.getPrimary().getRetentionLeases());
        }
        int numLeases = between(1, 100);
        IndexShard newPrimary = randomFrom(group.getReplicas());
        RetentionLeases latestRetentionLeasesOnNewPrimary = newPrimary.getRetentionLeases();
        for (int i = 0; i < numLeases; i++) {
            PlainActionFuture<ReplicationResponse> addLeaseFuture = new PlainActionFuture<>();
            group.addRetentionLease(Integer.toString(i), randomNonNegativeLong(), "test-" + i, addLeaseFuture);
            RetentionLeaseSyncAction.Request request = ((SyncRetentionLeasesResponse) addLeaseFuture.actionGet()).syncRequest;
            for (IndexShard replica : randomSubsetOf(group.getReplicas())) {
                group.executeRetentionLeasesSyncRequestOnReplica(request, replica);
                if (newPrimary == replica) {
                    latestRetentionLeasesOnNewPrimary = request.getRetentionLeases();
                }
            }
        }
        group.promoteReplicaToPrimary(newPrimary).get();
        // we need to make changes to retention leases to sync it to replicas
        // since we don't sync retention leases when promoting a new primary.
        PlainActionFuture<ReplicationResponse> newLeaseFuture = new PlainActionFuture<>();
        group.addRetentionLease("new-lease-after-promotion", randomNonNegativeLong(), "test", newLeaseFuture);
        RetentionLeases leasesOnPrimary = group.getPrimary().getRetentionLeases();
        assertThat(leasesOnPrimary.primaryTerm(), equalTo(group.getPrimary().getOperationPrimaryTerm()));
        assertThat(leasesOnPrimary.version(), equalTo(latestRetentionLeasesOnNewPrimary.version() + 1));
        assertThat(leasesOnPrimary.leases(), hasSize(latestRetentionLeasesOnNewPrimary.leases().size() + 1));
        RetentionLeaseSyncAction.Request request = ((SyncRetentionLeasesResponse) newLeaseFuture.actionGet()).syncRequest;
        for (IndexShard replica : group.getReplicas()) {
            group.executeRetentionLeasesSyncRequestOnReplica(request, replica);
        }
        for (IndexShard replica : group.getReplicas()) {
            assertThat(replica.getRetentionLeases(), equalTo(leasesOnPrimary));
        }
    }
}