Examples with TranslogDeletionPolicy org.elasticsearch.index.translog.TranslogDeletionPolicy used on opensource projects

Example 1 with TranslogDeletionPolicy

use of org.elasticsearch.index.translog.TranslogDeletionPolicy in project crate by crate.

the class InternalEngine method onSettingsChanged.

@Override
public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) {
    mergeScheduler.refreshConfig();
    // config().isEnableGcDeletes() or config.getGcDeletesInMillis() may have changed:
    maybePruneDeletes();
    final TranslogDeletionPolicy translogDeletionPolicy = translog.getDeletionPolicy();
    translogDeletionPolicy.setRetentionAgeInMillis(translogRetentionAge.millis());
    translogDeletionPolicy.setRetentionSizeInBytes(translogRetentionSize.getBytes());
    softDeletesPolicy.setRetentionOperations(softDeletesRetentionOps);
}

Example 2 with TranslogDeletionPolicy

use of org.elasticsearch.index.translog.TranslogDeletionPolicy in project crate by crate.

the class NoOpEngine method trimUnreferencedTranslogFiles.

/**
 * This implementation will trim existing translog files using a {@link TranslogDeletionPolicy}
 * that retains nothing but the last translog generation from safe commit.
 */
@Override
public void trimUnreferencedTranslogFiles() {
    final Store store = this.engineConfig.getStore();
    store.incRef();
    try (ReleasableLock lock = readLock.acquire()) {
        ensureOpen();
        final List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
        if (commits.size() == 1 && translogStats.getTranslogSizeInBytes() > translogStats.getUncommittedSizeInBytes()) {
            final Map<String, String> commitUserData = getLastCommittedSegmentInfos().getUserData();
            final String translogUuid = commitUserData.get(Translog.TRANSLOG_UUID_KEY);
            if (translogUuid == null) {
                throw new IllegalStateException("commit doesn't contain translog unique id");
            }
            final TranslogConfig translogConfig = engineConfig.getTranslogConfig();
            final long localCheckpoint = Long.parseLong(commitUserData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
            final TranslogDeletionPolicy translogDeletionPolicy = new TranslogDeletionPolicy(-1, -1, 0);
            translogDeletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint);
            try (Translog translog = new Translog(translogConfig, translogUuid, translogDeletionPolicy, engineConfig.getGlobalCheckpointSupplier(), engineConfig.getPrimaryTermSupplier(), seqNo -> {
            })) {
                translog.trimUnreferencedReaders();
                // refresh the translog stats
                this.translogStats = translog.stats();
                assert translog.currentFileGeneration() == translog.getMinFileGeneration() : "translog was not trimmed " + " current gen " + translog.currentFileGeneration() + " != min gen " + translog.getMinFileGeneration();
            }
        }
    } catch (final Exception e) {
        try {
            failEngine("translog trimming failed", e);
        } catch (Exception inner) {
            e.addSuppressed(inner);
        }
        throw new EngineException(shardId, "failed to trim translog", e);
    } finally {
        store.decRef();
    }
}

Example 3 with TranslogDeletionPolicy

use of org.elasticsearch.index.translog.TranslogDeletionPolicy in project crate by crate.

the class CombinedDeletionPolicyTests method testCheckUnreferencedCommits.

public void testCheckUnreferencedCommits() throws Exception {
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.UNASSIGNED_SEQ_NO);
    final SoftDeletesPolicy softDeletesPolicy = new SoftDeletesPolicy(globalCheckpoint::get, -1, 0, () -> RetentionLeases.EMPTY);
    final UUID translogUUID = UUID.randomUUID();
    final TranslogDeletionPolicy translogPolicy = createTranslogDeletionPolicy();
    CombinedDeletionPolicy indexPolicy = newCombinedDeletionPolicy(translogPolicy, softDeletesPolicy, globalCheckpoint);
    final List<IndexCommit> commitList = new ArrayList<>();
    int totalCommits = between(2, 20);
    long lastMaxSeqNo = between(1, 1000);
    long lastCheckpoint = randomLongBetween(-1, lastMaxSeqNo);
    for (int i = 0; i < totalCommits; i++) {
        lastMaxSeqNo += between(1, 10000);
        lastCheckpoint = randomLongBetween(lastCheckpoint, lastMaxSeqNo);
        commitList.add(mockIndexCommit(lastCheckpoint, lastMaxSeqNo, translogUUID));
    }
    int safeCommitIndex = randomIntBetween(0, commitList.size() - 1);
    globalCheckpoint.set(Long.parseLong(commitList.get(safeCommitIndex).getUserData().get(SequenceNumbers.MAX_SEQ_NO)));
    commitList.forEach(this::resetDeletion);
    indexPolicy.onCommit(commitList);
    if (safeCommitIndex == commitList.size() - 1) {
        // Safe commit is the last commit - no need to clean up
        assertThat(translogPolicy.getLocalCheckpointOfSafeCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
        assertThat(indexPolicy.hasUnreferencedCommits(), equalTo(false));
    } else {
        // Advanced but not enough for any commit after the safe commit becomes safe
        IndexCommit nextSafeCommit = commitList.get(safeCommitIndex + 1);
        globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), Long.parseLong(nextSafeCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)) - 1));
        assertFalse(indexPolicy.hasUnreferencedCommits());
        // Advanced enough for some index commit becomes safe
        globalCheckpoint.set(randomLongBetween(Long.parseLong(nextSafeCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)), lastMaxSeqNo));
        assertTrue(indexPolicy.hasUnreferencedCommits());
        // Advanced enough for the last commit becomes safe
        globalCheckpoint.set(randomLongBetween(lastMaxSeqNo, Long.MAX_VALUE));
        commitList.forEach(this::resetDeletion);
        indexPolicy.onCommit(commitList);
        // Safe commit is the last commit - no need to clean up
        assertThat(translogPolicy.getLocalCheckpointOfSafeCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
        assertThat(indexPolicy.hasUnreferencedCommits(), equalTo(false));
    }
}

Example 4 with TranslogDeletionPolicy

use of org.elasticsearch.index.translog.TranslogDeletionPolicy in project crate by crate.

the class CombinedDeletionPolicyTests method testAcquireIndexCommit.

public void testAcquireIndexCommit() throws Exception {
    final AtomicLong globalCheckpoint = new AtomicLong();
    final int extraRetainedOps = between(0, 100);
    final SoftDeletesPolicy softDeletesPolicy = new SoftDeletesPolicy(globalCheckpoint::get, -1, extraRetainedOps, () -> RetentionLeases.EMPTY);
    final UUID translogUUID = UUID.randomUUID();
    TranslogDeletionPolicy translogPolicy = createTranslogDeletionPolicy();
    CombinedDeletionPolicy indexPolicy = newCombinedDeletionPolicy(translogPolicy, softDeletesPolicy, globalCheckpoint);
    long lastMaxSeqNo = between(1, 1000);
    long lastCheckpoint = randomLongBetween(-1, lastMaxSeqNo);
    int safeIndex = 0;
    List<IndexCommit> commitList = new ArrayList<>();
    List<IndexCommit> snapshottingCommits = new ArrayList<>();
    final int iters = between(10, 100);
    for (int i = 0; i < iters; i++) {
        int newCommits = between(1, 10);
        for (int n = 0; n < newCommits; n++) {
            lastMaxSeqNo += between(1, 1000);
            lastCheckpoint = randomLongBetween(lastCheckpoint, lastMaxSeqNo);
            commitList.add(mockIndexCommit(lastCheckpoint, lastMaxSeqNo, translogUUID));
        }
        // Advance the global checkpoint to between [safeIndex, safeIndex + 1)
        safeIndex = randomIntBetween(safeIndex, commitList.size() - 1);
        long lower = Math.max(globalCheckpoint.get(), Long.parseLong(commitList.get(safeIndex).getUserData().get(SequenceNumbers.MAX_SEQ_NO)));
        long upper = safeIndex == commitList.size() - 1 ? lastMaxSeqNo : Long.parseLong(commitList.get(safeIndex + 1).getUserData().get(SequenceNumbers.MAX_SEQ_NO)) - 1;
        globalCheckpoint.set(randomLongBetween(lower, upper));
        commitList.forEach(this::resetDeletion);
        indexPolicy.onCommit(commitList);
        IndexCommit safeCommit = CombinedDeletionPolicy.findSafeCommitPoint(commitList, globalCheckpoint.get());
        assertThat(softDeletesPolicy.getMinRetainedSeqNo(), equalTo(Math.max(NO_OPS_PERFORMED, Math.min(getLocalCheckpoint(safeCommit) + 1, globalCheckpoint.get() + 1 - extraRetainedOps))));
        // Captures and releases some commits
        int captures = between(0, 5);
        for (int n = 0; n < captures; n++) {
            boolean safe = randomBoolean();
            final IndexCommit snapshot = indexPolicy.acquireIndexCommit(safe);
            expectThrows(UnsupportedOperationException.class, snapshot::delete);
            snapshottingCommits.add(snapshot);
            if (safe) {
                assertThat(snapshot.getUserData(), equalTo(commitList.get(safeIndex).getUserData()));
            } else {
                assertThat(snapshot.getUserData(), equalTo(commitList.get(commitList.size() - 1).getUserData()));
            }
        }
        final List<IndexCommit> releasingSnapshots = randomSubsetOf(snapshottingCommits);
        for (IndexCommit snapshot : releasingSnapshots) {
            snapshottingCommits.remove(snapshot);
            final long pendingSnapshots = snapshottingCommits.stream().filter(snapshot::equals).count();
            final IndexCommit lastCommit = commitList.get(commitList.size() - 1);
            safeCommit = CombinedDeletionPolicy.findSafeCommitPoint(commitList, globalCheckpoint.get());
            assertThat(indexPolicy.releaseCommit(snapshot), equalTo(pendingSnapshots == 0 && snapshot.equals(lastCommit) == false && snapshot.equals(safeCommit) == false));
        }
        // Snapshotting commits must not be deleted.
        snapshottingCommits.forEach(snapshot -> assertThat(snapshot.isDeleted(), equalTo(false)));
        // We don't need to retain translog for snapshotting commits.
        assertThat(translogPolicy.getLocalCheckpointOfSafeCommit(), equalTo(getLocalCheckpoint(commitList.get(safeIndex))));
        assertThat(softDeletesPolicy.getMinRetainedSeqNo(), equalTo(Math.max(NO_OPS_PERFORMED, Math.min(getLocalCheckpoint(commitList.get(safeIndex)) + 1, globalCheckpoint.get() + 1 - extraRetainedOps))));
    }
    snapshottingCommits.forEach(indexPolicy::releaseCommit);
    globalCheckpoint.set(randomLongBetween(lastMaxSeqNo, Long.MAX_VALUE));
    commitList.forEach(this::resetDeletion);
    indexPolicy.onCommit(commitList);
    for (int i = 0; i < commitList.size() - 1; i++) {
        assertThat(commitList.get(i).isDeleted(), equalTo(true));
    }
    assertThat(commitList.get(commitList.size() - 1).isDeleted(), equalTo(false));
    assertThat(translogPolicy.getLocalCheckpointOfSafeCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
    IndexCommit safeCommit = CombinedDeletionPolicy.findSafeCommitPoint(commitList, globalCheckpoint.get());
    assertThat(softDeletesPolicy.getMinRetainedSeqNo(), equalTo(Math.max(NO_OPS_PERFORMED, Math.min(getLocalCheckpoint(safeCommit) + 1, globalCheckpoint.get() + 1 - extraRetainedOps))));
}

Example 5 with TranslogDeletionPolicy

use of org.elasticsearch.index.translog.TranslogDeletionPolicy in project crate by crate.

the class CombinedDeletionPolicyTests method testKeepCommitsAfterGlobalCheckpoint.

public void testKeepCommitsAfterGlobalCheckpoint() throws Exception {
    final AtomicLong globalCheckpoint = new AtomicLong();
    final int extraRetainedOps = between(0, 100);
    final SoftDeletesPolicy softDeletesPolicy = new SoftDeletesPolicy(globalCheckpoint::get, NO_OPS_PERFORMED, extraRetainedOps, () -> RetentionLeases.EMPTY);
    TranslogDeletionPolicy translogPolicy = createTranslogDeletionPolicy();
    CombinedDeletionPolicy indexPolicy = newCombinedDeletionPolicy(translogPolicy, softDeletesPolicy, globalCheckpoint);
    final LongArrayList maxSeqNoList = new LongArrayList();
    final List<IndexCommit> commitList = new ArrayList<>();
    int totalCommits = between(2, 20);
    long lastMaxSeqNo = 0;
    long lastCheckpoint = lastMaxSeqNo;
    final UUID translogUUID = UUID.randomUUID();
    for (int i = 0; i < totalCommits; i++) {
        lastMaxSeqNo += between(1, 10000);
        lastCheckpoint = randomLongBetween(lastCheckpoint, lastMaxSeqNo);
        commitList.add(mockIndexCommit(lastCheckpoint, lastMaxSeqNo, translogUUID));
        maxSeqNoList.add(lastMaxSeqNo);
    }
    int keptIndex = randomInt(commitList.size() - 1);
    final long lower = maxSeqNoList.get(keptIndex);
    final long upper = keptIndex == commitList.size() - 1 ? Long.MAX_VALUE : Math.max(maxSeqNoList.get(keptIndex), maxSeqNoList.get(keptIndex + 1) - 1);
    globalCheckpoint.set(randomLongBetween(lower, upper));
    indexPolicy.onCommit(commitList);
    for (int i = 0; i < commitList.size(); i++) {
        if (i < keptIndex) {
            verify(commitList.get(i), times(1)).delete();
        } else {
            verify(commitList.get(i), never()).delete();
        }
    }
    assertThat(translogPolicy.getLocalCheckpointOfSafeCommit(), equalTo(getLocalCheckpoint(commitList.get(keptIndex))));
    assertThat(softDeletesPolicy.getMinRetainedSeqNo(), equalTo(Math.max(NO_OPS_PERFORMED, Math.min(getLocalCheckpoint(commitList.get(keptIndex)) + 1, globalCheckpoint.get() + 1 - extraRetainedOps))));
}