Examples with RetentionLeases org.elasticsearch.index.seqno.RetentionLeases used on opensource projects

Example 6 with RetentionLeases

use of org.elasticsearch.index.seqno.RetentionLeases in project crate by crate.

the class RecoverySourceHandlerTests method testThrowExceptionOnPrimaryRelocatedBeforePhase1Started.

@Test
public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Started() throws IOException {
    final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service);
    final StartRecoveryRequest request = getStartRecoveryRequest();
    final IndexShard shard = mock(IndexShard.class);
    when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class));
    when(shard.isRelocatedPrimary()).thenReturn(true);
    when(shard.acquireSafeIndexCommit()).thenReturn(mock(Engine.IndexCommitRef.class));
    doAnswer(invocation -> {
        ((ActionListener<Releasable>) invocation.getArguments()[0]).onResponse(() -> {
        });
        return null;
    }).when(shard).acquirePrimaryOperationPermit(any(), anyString(), anyObject());
    final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("test").settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 5)).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5)).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random())).put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random())));
    if (randomBoolean()) {
        indexMetadata.state(IndexMetadata.State.CLOSE);
    }
    when(shard.indexSettings()).thenReturn(new IndexSettings(indexMetadata.build(), Settings.EMPTY));
    final AtomicBoolean phase1Called = new AtomicBoolean();
    final AtomicBoolean prepareTargetForTranslogCalled = new AtomicBoolean();
    final AtomicBoolean phase2Called = new AtomicBoolean();
    final RecoverySourceHandler handler = new RecoverySourceHandler(shard, mock(RecoveryTargetHandler.class), threadPool, request, Math.toIntExact(recoverySettings.getChunkSize().getBytes()), between(1, 8), between(1, 8)) {

        @Override
        void phase1(IndexCommit snapshot, long startingSeqNo, IntSupplier translogOps, ActionListener<SendFileResult> listener) {
            phase1Called.set(true);
            super.phase1(snapshot, startingSeqNo, translogOps, listener);
        }

        @Override
        void prepareTargetForTranslog(int totalTranslogOps, ActionListener<TimeValue> listener) {
            prepareTargetForTranslogCalled.set(true);
            super.prepareTargetForTranslog(totalTranslogOps, listener);
        }

        @Override
        void phase2(long startingSeqNo, long endingSeqNo, Translog.Snapshot snapshot, long maxSeenAutoIdTimestamp, long maxSeqNoOfUpdatesOrDeletes, RetentionLeases retentionLeases, long mappingVersion, ActionListener<SendSnapshotResult> listener) throws IOException {
            phase2Called.set(true);
            super.phase2(startingSeqNo, endingSeqNo, snapshot, maxSeenAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, mappingVersion, listener);
        }
    };
    PlainActionFuture<RecoveryResponse> future = new PlainActionFuture<>();
    expectThrows(IndexShardRelocatedException.class, () -> {
        handler.recoverToTarget(future);
        future.actionGet();
    });
    assertFalse(phase1Called.get());
    assertFalse(prepareTargetForTranslogCalled.get());
    assertFalse(phase2Called.get());
}

Example 7 with RetentionLeases

use of org.elasticsearch.index.seqno.RetentionLeases in project crate by crate.

the class SoftDeletesPolicyTests method testWhenLocalCheckpointOfSafeCommitDictatesThePolicy.

@Test
public void testWhenLocalCheckpointOfSafeCommitDictatesThePolicy() {
    final int retentionOperations = randomIntBetween(0, 1024);
    final long localCheckpointOfSafeCommit = randomLongBetween(-1, Long.MAX_VALUE - retentionOperations - 1);
    final AtomicLong globalCheckpoint = new AtomicLong(randomLongBetween(Math.max(0, localCheckpointOfSafeCommit + retentionOperations), Long.MAX_VALUE - 1));
    final Collection<RetentionLease> leases = new ArrayList<>();
    final int numberOfLeases = randomIntBetween(0, 16);
    for (int i = 0; i < numberOfLeases; i++) {
        leases.add(new RetentionLease(Integer.toString(i), // leases are for more than the local checkpoint
        randomLongBetween(1 + localCheckpointOfSafeCommit + 1, Long.MAX_VALUE), randomNonNegativeLong(), "test"));
    }
    final long primaryTerm = randomNonNegativeLong();
    final long version = randomNonNegativeLong();
    final Supplier<RetentionLeases> leasesSupplier = () -> new RetentionLeases(primaryTerm, version, Collections.unmodifiableCollection(new ArrayList<>(leases)));
    final SoftDeletesPolicy policy = new SoftDeletesPolicy(globalCheckpoint::get, 0, retentionOperations, leasesSupplier);
    policy.setLocalCheckpointOfSafeCommit(localCheckpointOfSafeCommit);
    assertThat(policy.getMinRetainedSeqNo(), equalTo(1 + localCheckpointOfSafeCommit));
}

Example 8 with RetentionLeases

use of org.elasticsearch.index.seqno.RetentionLeases in project crate by crate.

the class SoftDeletesPolicyTests method testWhenGlobalCheckpointDictatesThePolicy.

@Test
public void testWhenGlobalCheckpointDictatesThePolicy() {
    final int retentionOperations = randomIntBetween(0, 1024);
    final AtomicLong globalCheckpoint = new AtomicLong(randomLongBetween(0, Long.MAX_VALUE - 2));
    final Collection<RetentionLease> leases = new ArrayList<>();
    final int numberOfLeases = randomIntBetween(0, 16);
    for (int i = 0; i < numberOfLeases; i++) {
        // setup leases where the minimum retained sequence number is more than the policy dictated by the global checkpoint
        leases.add(new RetentionLease(Integer.toString(i), randomLongBetween(1 + globalCheckpoint.get() - retentionOperations + 1, Long.MAX_VALUE), randomNonNegativeLong(), "test"));
    }
    final long primaryTerm = randomNonNegativeLong();
    final long version = randomNonNegativeLong();
    final Supplier<RetentionLeases> leasesSupplier = () -> new RetentionLeases(primaryTerm, version, Collections.unmodifiableCollection(new ArrayList<>(leases)));
    final SoftDeletesPolicy policy = new SoftDeletesPolicy(globalCheckpoint::get, 0, retentionOperations, leasesSupplier);
    // set the local checkpoint of the safe commit to more than the policy dicated by the global checkpoint
    final long localCheckpointOfSafeCommit = randomLongBetween(1 + globalCheckpoint.get() - retentionOperations + 1, Long.MAX_VALUE);
    policy.setLocalCheckpointOfSafeCommit(localCheckpointOfSafeCommit);
    assertThat(policy.getMinRetainedSeqNo(), equalTo(1 + globalCheckpoint.get() - retentionOperations));
}

Example 9 with RetentionLeases

use of org.elasticsearch.index.seqno.RetentionLeases in project crate by crate.

the class IndexShardRetentionLeaseTests method runExpirationTest.

private void runExpirationTest(final boolean primary) throws IOException {
    final long retentionLeaseMillis = randomLongBetween(1, TimeValue.timeValueHours(12).millis());
    final Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING.getKey(), TimeValue.timeValueMillis(retentionLeaseMillis).getStringRep()).build();
    // current time is mocked through the thread pool
    final IndexShard indexShard = newStartedShard(primary, settings, new InternalEngineFactory());
    final long primaryTerm = indexShard.getOperationPrimaryTerm();
    try {
        final long[] retainingSequenceNumbers = new long[1];
        retainingSequenceNumbers[0] = randomLongBetween(0, Long.MAX_VALUE);
        final long initialVersion;
        if (primary) {
            initialVersion = 2;
            indexShard.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> {
            }));
        } else {
            initialVersion = 3;
            final RetentionLeases retentionLeases = new RetentionLeases(primaryTerm, initialVersion, Arrays.asList(peerRecoveryRetentionLease(indexShard), new RetentionLease("0", retainingSequenceNumbers[0], currentTimeMillis.get(), "test-0")));
            indexShard.updateRetentionLeasesOnReplica(retentionLeases);
        }
        {
            final RetentionLeases retentionLeases = indexShard.getEngine().config().retentionLeasesSupplier().get();
            assertThat(retentionLeases.version(), equalTo(initialVersion));
            assertThat(retentionLeases.leases(), hasSize(2));
            final RetentionLease retentionLease = retentionLeases.get("0");
            assertThat(retentionLease.timestamp(), equalTo(currentTimeMillis.get()));
            assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, initialVersion, primary, false);
        }
        // renew the lease
        currentTimeMillis.set(currentTimeMillis.get() + randomLongBetween(0, 1024));
        retainingSequenceNumbers[0] = randomLongBetween(retainingSequenceNumbers[0], Long.MAX_VALUE);
        if (primary) {
            indexShard.renewRetentionLease("0", retainingSequenceNumbers[0], "test-0");
        } else {
            final RetentionLeases retentionLeases = new RetentionLeases(primaryTerm, initialVersion + 1, Arrays.asList(peerRecoveryRetentionLease(indexShard), new RetentionLease("0", retainingSequenceNumbers[0], currentTimeMillis.get(), "test-0")));
            indexShard.updateRetentionLeasesOnReplica(retentionLeases);
        }
        {
            final RetentionLeases retentionLeases = indexShard.getEngine().config().retentionLeasesSupplier().get();
            assertThat(retentionLeases.version(), equalTo(initialVersion + 1));
            assertThat(retentionLeases.leases(), hasSize(2));
            final RetentionLease retentionLease = retentionLeases.get("0");
            assertThat(retentionLease.timestamp(), equalTo(currentTimeMillis.get()));
            assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, initialVersion + 1, primary, false);
        }
        // now force the lease to expire
        currentTimeMillis.set(currentTimeMillis.get() + randomLongBetween(retentionLeaseMillis, Long.MAX_VALUE - currentTimeMillis.get()));
        if (primary) {
            assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, initialVersion + 1, true, false);
            assertRetentionLeases(indexShard, 0, new long[0], primaryTerm, initialVersion + 2, true, true);
        } else {
            assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, initialVersion + 1, false, false);
        }
    } finally {
        closeShards(indexShard);
    }
}

Example 10 with RetentionLeases

use of org.elasticsearch.index.seqno.RetentionLeases in project crate by crate.

the class IndexShardRetentionLeaseTests method assertRetentionLeases.

private void assertRetentionLeases(final IndexShard indexShard, final int size, final long[] minimumRetainingSequenceNumbers, final long primaryTerm, final long version, final boolean primary, final boolean expireLeases) {
    assertTrue(expireLeases == false || primary);
    final RetentionLeases retentionLeases;
    if (expireLeases == false) {
        if (randomBoolean()) {
            retentionLeases = indexShard.getRetentionLeases();
        } else {
            final Tuple<Boolean, RetentionLeases> tuple = indexShard.getRetentionLeases(false);
            assertFalse(tuple.v1());
            retentionLeases = tuple.v2();
        }
    } else {
        final Tuple<Boolean, RetentionLeases> tuple = indexShard.getRetentionLeases(true);
        assertTrue(tuple.v1());
        retentionLeases = tuple.v2();
    }
    assertRetentionLeases(retentionLeases, size, minimumRetainingSequenceNumbers, primaryTerm, version);
}