Examples with RetentionLease org.elasticsearch.index.seqno.RetentionLease used on opensource projects

Example 11 with RetentionLease

use of org.elasticsearch.index.seqno.RetentionLease in project crate by crate.

the class SoftDeletesPolicy method getMinRetainedSeqNo.

/**
 * Returns the min seqno that is retained in the Lucene index.
 * Operations whose seq# is least this value should exist in the Lucene index.
 */
synchronized long getMinRetainedSeqNo() {
    /*
         * When an engine is flushed, we need to provide it the latest collection of retention leases even when the soft deletes policy is
         * locked for peer recovery.
         */
    final RetentionLeases retentionLeases = retentionLeasesSupplier.get();
    // do not advance if the retention lock is held
    if (retentionLockCount == 0) {
        /*
             * This policy retains operations for two purposes: peer-recovery and querying changes history.
             *  - Peer-recovery is driven by the local checkpoint of the safe commit. In peer-recovery, the primary transfers a safe commit,
             *    then sends operations after the local checkpoint of that commit. This requires keeping all ops after
             *    localCheckpointOfSafeCommit.
             *  - Changes APIs are driven by a combination of the global checkpoint, retention operations, and retention leases. Here we
             *    prefer using the global checkpoint instead of the maximum sequence number because only operations up to the global
             *    checkpoint are exposed in the the changes APIs.
             */
        // calculate the minimum sequence number to retain based on retention leases
        final long minimumRetainingSequenceNumber = retentionLeases.leases().stream().mapToLong(RetentionLease::retainingSequenceNumber).min().orElse(Long.MAX_VALUE);
        /*
             * The minimum sequence number to retain is the minimum of the minimum based on retention leases, and the number of operations
             * below the global checkpoint to retain (index.soft_deletes.retention.operations). The additional increments on the global
             * checkpoint and the local checkpoint of the safe commit are due to the fact that we want to retain all operations above
             * those checkpoints.
             */
        final long minSeqNoForQueryingChanges = Math.min(1 + globalCheckpointSupplier.getAsLong() - retentionOperations, minimumRetainingSequenceNumber);
        final long minSeqNoToRetain = Math.min(minSeqNoForQueryingChanges, 1 + localCheckpointOfSafeCommit);
        /*
             * We take the maximum as minSeqNoToRetain can go backward as the retention operations value can be changed in settings, or from
             * the addition of leases with a retaining sequence number lower than previous retaining sequence numbers.
             */
        minRetainedSeqNo = Math.max(minRetainedSeqNo, minSeqNoToRetain);
    }
    return minRetainedSeqNo;
}

Example 12 with RetentionLease

use of org.elasticsearch.index.seqno.RetentionLease in project crate by crate.

the class SoftDeletesPolicyTests method testWhenRetentionLeasesDictateThePolicy.

@Test
public void testWhenRetentionLeasesDictateThePolicy() {
    final int retentionOperations = randomIntBetween(0, 1024);
    final Collection<RetentionLease> leases = new ArrayList<>();
    final int numberOfLeases = randomIntBetween(1, 16);
    for (int i = 0; i < numberOfLeases; i++) {
        leases.add(new RetentionLease(Integer.toString(i), randomLongBetween(0, Long.MAX_VALUE - retentionOperations - 1), randomNonNegativeLong(), "test"));
    }
    final OptionalLong minimumRetainingSequenceNumber = leases.stream().mapToLong(RetentionLease::retainingSequenceNumber).min();
    assert minimumRetainingSequenceNumber.isPresent() : leases;
    final long localCheckpointOfSafeCommit = randomLongBetween(minimumRetainingSequenceNumber.getAsLong(), Long.MAX_VALUE - 1);
    final AtomicLong globalCheckpoint = new AtomicLong(randomLongBetween(minimumRetainingSequenceNumber.getAsLong() + retentionOperations, Long.MAX_VALUE - 1));
    final long primaryTerm = randomNonNegativeLong();
    final long version = randomNonNegativeLong();
    final Supplier<RetentionLeases> leasesSupplier = () -> new RetentionLeases(primaryTerm, version, Collections.unmodifiableCollection(new ArrayList<>(leases)));
    final SoftDeletesPolicy policy = new SoftDeletesPolicy(globalCheckpoint::get, 0, retentionOperations, leasesSupplier);
    policy.setLocalCheckpointOfSafeCommit(localCheckpointOfSafeCommit);
    assertThat(policy.getMinRetainedSeqNo(), equalTo(minimumRetainingSequenceNumber.getAsLong()));
}

Example 13 with RetentionLease

use of org.elasticsearch.index.seqno.RetentionLease in project crate by crate.

the class StoreTests method testStreamStoreFilesMetadata.

public void testStreamStoreFilesMetadata() throws Exception {
    Store.MetadataSnapshot metadataSnapshot = createMetadataSnapshot();
    int numOfLeases = randomIntBetween(0, 10);
    List<RetentionLease> peerRecoveryRetentionLeases = new ArrayList<>();
    for (int i = 0; i < numOfLeases; i++) {
        peerRecoveryRetentionLeases.add(new RetentionLease(ReplicationTracker.getPeerRecoveryRetentionLeaseId(UUIDs.randomBase64UUID()), randomNonNegativeLong(), randomNonNegativeLong(), ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE));
    }
    TransportNodesListShardStoreMetadata.StoreFilesMetadata outStoreFileMetadata = new TransportNodesListShardStoreMetadata.StoreFilesMetadata(new ShardId("test", "_na_", 0), metadataSnapshot, peerRecoveryRetentionLeases);
    ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
    OutputStreamStreamOutput out = new OutputStreamStreamOutput(outBuffer);
    org.elasticsearch.Version targetNodeVersion = randomVersion(random());
    out.setVersion(targetNodeVersion);
    outStoreFileMetadata.writeTo(out);
    ByteArrayInputStream inBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
    InputStreamStreamInput in = new InputStreamStreamInput(inBuffer);
    in.setVersion(targetNodeVersion);
    TransportNodesListShardStoreMetadata.StoreFilesMetadata inStoreFileMetadata = new TransportNodesListShardStoreMetadata.StoreFilesMetadata(in);
    Iterator<StoreFileMetadata> outFiles = outStoreFileMetadata.iterator();
    for (StoreFileMetadata inFile : inStoreFileMetadata) {
        assertThat(inFile.name(), equalTo(outFiles.next().name()));
    }
    assertThat(outStoreFileMetadata.syncId(), equalTo(inStoreFileMetadata.syncId()));
    assertThat(outStoreFileMetadata.peerRecoveryRetentionLeases(), equalTo(peerRecoveryRetentionLeases));
}

Example 14 with RetentionLease

use of org.elasticsearch.index.seqno.RetentionLease in project crate by crate.

the class IndexShardRetentionLeaseTests method assertRetentionLeases.

private void assertRetentionLeases(final RetentionLeases retentionLeases, final int size, final long[] minimumRetainingSequenceNumbers, final long primaryTerm, final long version) {
    assertThat(retentionLeases.primaryTerm(), equalTo(primaryTerm));
    assertThat(retentionLeases.version(), equalTo(version));
    final Map<String, RetentionLease> idToRetentionLease = new HashMap<>();
    for (final RetentionLease retentionLease : retentionLeases.leases()) {
        if (ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE.equals(retentionLease.source()) == false) {
            idToRetentionLease.put(retentionLease.id(), retentionLease);
        }
    }
    assertThat(idToRetentionLease.entrySet(), hasSize(size));
    for (int i = 0; i < size; i++) {
        assertThat(idToRetentionLease.keySet(), hasItem(Integer.toString(i)));
        final RetentionLease retentionLease = idToRetentionLease.get(Integer.toString(i));
        assertThat(retentionLease.retainingSequenceNumber(), equalTo(minimumRetainingSequenceNumbers[i]));
        assertThat(retentionLease.source(), equalTo("test-" + i));
    }
}

Example 15 with RetentionLease

use of org.elasticsearch.index.seqno.RetentionLease in project crate by crate.

the class InternalEngineTests method testKeepMinRetainedSeqNoByMergePolicy.

@Test
public void testKeepMinRetainedSeqNoByMergePolicy() throws IOException {
    IOUtils.close(engine, store);
    Settings.Builder settings = Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), randomLongBetween(0, 10));
    final IndexMetadata indexMetadata = IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build();
    final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetadata);
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    final long primaryTerm = randomLongBetween(1, Long.MAX_VALUE);
    final AtomicLong retentionLeasesVersion = new AtomicLong();
    final AtomicReference<RetentionLeases> retentionLeasesHolder = new AtomicReference<>(new RetentionLeases(primaryTerm, retentionLeasesVersion.get(), Collections.emptyList()));
    final List<Engine.Operation> operations = generateSingleDocHistory(true, randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), false, 2, 10, 300, "2");
    Randomness.shuffle(operations);
    Set<Long> existingSeqNos = new HashSet<>();
    store = createStore();
    engine = createEngine(config(indexSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get, retentionLeasesHolder::get));
    assertThat(engine.getMinRetainedSeqNo(), equalTo(0L));
    long lastMinRetainedSeqNo = engine.getMinRetainedSeqNo();
    for (Engine.Operation op : operations) {
        final Engine.Result result;
        if (op instanceof Engine.Index) {
            result = engine.index((Engine.Index) op);
        } else {
            result = engine.delete((Engine.Delete) op);
        }
        existingSeqNos.add(result.getSeqNo());
        if (randomBoolean()) {
            // advance persisted local checkpoint
            engine.syncTranslog();
            assertEquals(engine.getProcessedLocalCheckpoint(), engine.getPersistedLocalCheckpoint());
            globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpointTracker().getPersistedCheckpoint()));
        }
        if (randomBoolean()) {
            retentionLeasesVersion.incrementAndGet();
            final int length = randomIntBetween(0, 8);
            final List<RetentionLease> leases = new ArrayList<>(length);
            for (int i = 0; i < length; i++) {
                final String id = randomAlphaOfLength(8);
                final long retainingSequenceNumber = randomLongBetween(0, Math.max(0, globalCheckpoint.get()));
                final long timestamp = randomLongBetween(0L, Long.MAX_VALUE);
                final String source = randomAlphaOfLength(8);
                leases.add(new RetentionLease(id, retainingSequenceNumber, timestamp, source));
            }
            retentionLeasesHolder.set(new RetentionLeases(primaryTerm, retentionLeasesVersion.get(), leases));
        }
        if (rarely()) {
            settings.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), randomLongBetween(0, 10));
            indexSettings.updateIndexMetadata(IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build());
            engine.onSettingsChanged(indexSettings.getTranslogRetentionAge(), indexSettings.getTranslogRetentionSize(), indexSettings.getSoftDeleteRetentionOperations());
        }
        if (rarely()) {
            engine.refresh("test");
        }
        if (rarely()) {
            engine.flush(true, true);
            assertThat(Long.parseLong(engine.getLastCommittedSegmentInfos().userData.get(Engine.MIN_RETAINED_SEQNO)), equalTo(engine.getMinRetainedSeqNo()));
        }
        if (rarely()) {
            engine.forceMerge(randomBoolean(), 1, false, false, false, UUIDs.randomBase64UUID());
        }
        try (Closeable ignored = engine.acquireHistoryRetentionLock(Engine.HistorySource.INDEX)) {
            long minRetainSeqNos = engine.getMinRetainedSeqNo();
            assertThat(minRetainSeqNos, lessThanOrEqualTo(globalCheckpoint.get() + 1));
            Long[] expectedOps = existingSeqNos.stream().filter(seqno -> seqno >= minRetainSeqNos).toArray(Long[]::new);
            Set<Long> actualOps = readAllOperationsInLucene(engine, createMapperService("test")).stream().map(Translog.Operation::seqNo).collect(Collectors.toSet());
            assertThat(actualOps, containsInAnyOrder(expectedOps));
        }
        try (Engine.IndexCommitRef commitRef = engine.acquireSafeIndexCommit()) {
            IndexCommit safeCommit = commitRef.getIndexCommit();
            if (safeCommit.getUserData().containsKey(Engine.MIN_RETAINED_SEQNO)) {
                lastMinRetainedSeqNo = Long.parseLong(safeCommit.getUserData().get(Engine.MIN_RETAINED_SEQNO));
            }
        }
    }
    if (randomBoolean()) {
        engine.close();
    } else {
        engine.flushAndClose();
    }
    try (InternalEngine recoveringEngine = new InternalEngine(engine.config())) {
        assertThat(recoveringEngine.getMinRetainedSeqNo(), equalTo(lastMinRetainedSeqNo));
    }
}