Examples with RetentionLease org.elasticsearch.index.seqno.RetentionLease used on opensource projects

Example 1 with RetentionLease

use of org.elasticsearch.index.seqno.RetentionLease in project crate by crate.

the class ReplicaShardAllocatorTests method testIgnoreRetentionLeaseIfCopyIsEmpty.

public void testIgnoreRetentionLeaseIfCopyIsEmpty() {
    RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders());
    long retainingSeqNo = randomLongBetween(1, Long.MAX_VALUE);
    List<RetentionLease> retentionLeases = new ArrayList<>();
    retentionLeases.add(newRetentionLease(node1, retainingSeqNo));
    retentionLeases.add(newRetentionLease(node2, randomLongBetween(0, retainingSeqNo)));
    if (randomBoolean()) {
        retentionLeases.add(newRetentionLease(node3, randomLongBetween(0, retainingSeqNo)));
    }
    testAllocator.addData(node1, retentionLeases, randomSyncId(), new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    // has retention lease but store is empty
    testAllocator.addData(node2, null);
    testAllocator.addData(node3, randomSyncId(), new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    testAllocator.allocateUnassigned(allocation);
    assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
    assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node3.getId()));
}

Example 2 with RetentionLease

use of org.elasticsearch.index.seqno.RetentionLease in project crate by crate.

the class ReplicaShardAllocatorTests method testNotCancellingRecoveryIfCurrentRecoveryHasRetentionLease.

public void testNotCancellingRecoveryIfCurrentRecoveryHasRetentionLease() {
    RoutingAllocation allocation = onePrimaryOnNode1And1ReplicaRecovering(yesAllocationDeciders());
    List<RetentionLease> peerRecoveryRetentionLeasesOnPrimary = new ArrayList<>();
    long retainingSeqNo = randomLongBetween(1, Long.MAX_VALUE);
    peerRecoveryRetentionLeasesOnPrimary.add(newRetentionLease(node1, retainingSeqNo));
    peerRecoveryRetentionLeasesOnPrimary.add(newRetentionLease(node2, randomLongBetween(1, retainingSeqNo)));
    if (randomBoolean()) {
        peerRecoveryRetentionLeasesOnPrimary.add(newRetentionLease(node3, randomLongBetween(0, retainingSeqNo)));
    }
    testAllocator.addData(node1, peerRecoveryRetentionLeasesOnPrimary, "MATCH", new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    testAllocator.addData(node2, randomSyncId(), new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    testAllocator.addData(node3, randomSyncId(), new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    testAllocator.processExistingRecoveries(allocation);
    assertThat(allocation.routingNodesChanged(), equalTo(false));
    assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(0));
}

Example 3 with RetentionLease

use of org.elasticsearch.index.seqno.RetentionLease in project crate by crate.

the class ReplicaShardAllocatorTests method testDoNotCancelForBrokenNode.

public void testDoNotCancelForBrokenNode() {
    Set<String> failedNodes = new HashSet<>();
    failedNodes.add(node3.getId());
    if (randomBoolean()) {
        failedNodes.add("node4");
    }
    UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, null, null, randomIntBetween(failedNodes.size(), 10), System.nanoTime(), System.currentTimeMillis(), false, UnassignedInfo.AllocationStatus.NO_ATTEMPT, failedNodes);
    RoutingAllocation allocation = onePrimaryOnNode1And1ReplicaRecovering(yesAllocationDeciders(), unassignedInfo);
    long retainingSeqNoOnPrimary = randomLongBetween(0, Long.MAX_VALUE);
    List<RetentionLease> retentionLeases = Arrays.asList(newRetentionLease(node1, retainingSeqNoOnPrimary), newRetentionLease(node3, retainingSeqNoOnPrimary));
    testAllocator.addData(node1, retentionLeases, "MATCH", new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION)).addData(node2, randomSyncId(), new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION)).addData(node3, randomSyncId(), new StoreFileMetadata("file1", 10, "MATCH_CHECKSUM", MIN_SUPPORTED_LUCENE_VERSION));
    testAllocator.processExistingRecoveries(allocation);
    assertThat(allocation.routingNodesChanged(), equalTo(false));
    assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED), empty());
}

Example 4 with RetentionLease

use of org.elasticsearch.index.seqno.RetentionLease in project crate by crate.

the class RecoverySourceHandlerTests method testCancelRecoveryDuringPhase1.

@Test
public void testCancelRecoveryDuringPhase1() throws Exception {
    Store store = newStore(createTempDir("source"), false);
    IndexShard shard = mock(IndexShard.class);
    when(shard.store()).thenReturn(store);
    Directory dir = store.directory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
    int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
        writer.addDocument(document);
    }
    writer.commit();
    writer.close();
    AtomicBoolean wasCancelled = new AtomicBoolean();
    SetOnce<Runnable> cancelRecovery = new SetOnce<>();
    final TestRecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() {

        @Override
        public void receiveFileInfo(List<String> phase1FileNames, List<Long> phase1FileSizes, List<String> phase1ExistingFileNames, List<Long> phase1ExistingFileSizes, int totalTranslogOps, ActionListener<Void> listener) {
            recoveryExecutor.execute(() -> listener.onResponse(null));
            if (randomBoolean()) {
                wasCancelled.set(true);
                cancelRecovery.get().run();
            }
        }

        @Override
        public void writeFileChunk(StoreFileMetadata md, long position, BytesReference content, boolean lastChunk, int totalTranslogOps, ActionListener<Void> listener) {
            recoveryExecutor.execute(() -> listener.onResponse(null));
            if (rarely()) {
                wasCancelled.set(true);
                cancelRecovery.get().run();
            }
        }

        @Override
        public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetadata, ActionListener<Void> listener) {
            recoveryExecutor.execute(() -> listener.onResponse(null));
            if (randomBoolean()) {
                wasCancelled.set(true);
                cancelRecovery.get().run();
            }
        }
    };
    StartRecoveryRequest startRecoveryRequest = getStartRecoveryRequest();
    final RecoverySourceHandler handler = new RecoverySourceHandler(shard, recoveryTarget, threadPool, startRecoveryRequest, between(1, 16), between(1, 4), between(1, 4)) {

        @Override
        void createRetentionLease(long startingSeqNo, ActionListener<RetentionLease> listener) {
            final String leaseId = ReplicationTracker.getPeerRecoveryRetentionLeaseId(startRecoveryRequest.targetNode().getId());
            listener.onResponse(new RetentionLease(leaseId, startingSeqNo, threadPool.absoluteTimeInMillis(), ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE));
        }
    };
    cancelRecovery.set(() -> handler.cancel("test"));
    final StepListener<RecoverySourceHandler.SendFileResult> phase1Listener = new StepListener<>();
    try {
        final CountDownLatch latch = new CountDownLatch(1);
        handler.phase1(DirectoryReader.listCommits(dir).get(0), 0, () -> 0, new LatchedActionListener<>(phase1Listener, latch));
        latch.await();
        phase1Listener.result();
    } catch (Exception e) {
        assertTrue(wasCancelled.get());
        assertNotNull(ExceptionsHelper.unwrap(e, CancellableThreads.ExecutionCancelledException.class));
    }
    store.close();
}

Example 5 with RetentionLease

use of org.elasticsearch.index.seqno.RetentionLease in project crate by crate.

the class SoftDeletesPolicyTests method testWhenLocalCheckpointOfSafeCommitDictatesThePolicy.

@Test
public void testWhenLocalCheckpointOfSafeCommitDictatesThePolicy() {
    final int retentionOperations = randomIntBetween(0, 1024);
    final long localCheckpointOfSafeCommit = randomLongBetween(-1, Long.MAX_VALUE - retentionOperations - 1);
    final AtomicLong globalCheckpoint = new AtomicLong(randomLongBetween(Math.max(0, localCheckpointOfSafeCommit + retentionOperations), Long.MAX_VALUE - 1));
    final Collection<RetentionLease> leases = new ArrayList<>();
    final int numberOfLeases = randomIntBetween(0, 16);
    for (int i = 0; i < numberOfLeases; i++) {
        leases.add(new RetentionLease(Integer.toString(i), // leases are for more than the local checkpoint
        randomLongBetween(1 + localCheckpointOfSafeCommit + 1, Long.MAX_VALUE), randomNonNegativeLong(), "test"));
    }
    final long primaryTerm = randomNonNegativeLong();
    final long version = randomNonNegativeLong();
    final Supplier<RetentionLeases> leasesSupplier = () -> new RetentionLeases(primaryTerm, version, Collections.unmodifiableCollection(new ArrayList<>(leases)));
    final SoftDeletesPolicy policy = new SoftDeletesPolicy(globalCheckpoint::get, 0, retentionOperations, leasesSupplier);
    policy.setLocalCheckpointOfSafeCommit(localCheckpointOfSafeCommit);
    assertThat(policy.getMinRetainedSeqNo(), equalTo(1 + localCheckpointOfSafeCommit));
}