Examples with RetentionLeases org.elasticsearch.index.seqno.RetentionLeases used on opensource projects

Example 16 with RetentionLeases

use of org.elasticsearch.index.seqno.RetentionLeases in project crate by crate.

the class IndexShardRetentionLeaseTests method testPersistence.

public void testPersistence() throws IOException {
    final Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING.getKey(), Long.MAX_VALUE, TimeUnit.NANOSECONDS).build();
    final IndexShard indexShard = newStartedShard(true, settings, new InternalEngineFactory());
    try {
        final int length = randomIntBetween(0, 8);
        final long[] minimumRetainingSequenceNumbers = new long[length];
        for (int i = 0; i < length; i++) {
            minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
            currentTimeMillis.set(TimeUnit.NANOSECONDS.toMillis(randomNonNegativeLong()));
            indexShard.addRetentionLease(Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> {
            }));
        }
        currentTimeMillis.set(TimeUnit.NANOSECONDS.toMillis(Long.MAX_VALUE));
        // force the retention leases to persist
        indexShard.persistRetentionLeases();
        // the written retention leases should equal our current retention leases
        final RetentionLeases retentionLeases = indexShard.getEngine().config().retentionLeasesSupplier().get();
        final RetentionLeases writtenRetentionLeases = indexShard.loadRetentionLeases();
        assertThat(writtenRetentionLeases.version(), equalTo(1L + length));
        assertThat(writtenRetentionLeases.leases(), contains(retentionLeases.leases().toArray(new RetentionLease[0])));
        // when we recover, we should recover the retention leases
        final IndexShard recoveredShard = reinitShard(indexShard, ShardRoutingHelper.initWithSameId(indexShard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE));
        try {
            recoverShardFromStore(recoveredShard);
            final RetentionLeases recoveredRetentionLeases = recoveredShard.getEngine().config().retentionLeasesSupplier().get();
            assertThat(recoveredRetentionLeases.version(), equalTo(1L + length));
            assertThat(recoveredRetentionLeases.leases(), contains(retentionLeases.leases().toArray(new RetentionLease[0])));
        } finally {
            closeShards(recoveredShard);
        }
        // we should not recover retention leases when force-allocating a stale primary
        final IndexShard forceRecoveredShard = reinitShard(indexShard, ShardRoutingHelper.initWithSameId(indexShard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE));
        try {
            recoverShardFromStore(forceRecoveredShard);
            final RetentionLeases recoveredRetentionLeases = forceRecoveredShard.getEngine().config().retentionLeasesSupplier().get();
            assertThat(recoveredRetentionLeases.leases(), hasSize(1));
            assertThat(recoveredRetentionLeases.leases().iterator().next().id(), equalTo(ReplicationTracker.getPeerRecoveryRetentionLeaseId(indexShard.routingEntry())));
            assertThat(recoveredRetentionLeases.version(), equalTo(1L));
        } finally {
            closeShards(forceRecoveredShard);
        }
    } finally {
        closeShards(indexShard);
    }
}

Example 17 with RetentionLeases

use of org.elasticsearch.index.seqno.RetentionLeases in project crate by crate.

the class InternalEngineTests method testKeepMinRetainedSeqNoByMergePolicy.

@Test
public void testKeepMinRetainedSeqNoByMergePolicy() throws IOException {
    IOUtils.close(engine, store);
    Settings.Builder settings = Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), randomLongBetween(0, 10));
    final IndexMetadata indexMetadata = IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build();
    final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetadata);
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    final long primaryTerm = randomLongBetween(1, Long.MAX_VALUE);
    final AtomicLong retentionLeasesVersion = new AtomicLong();
    final AtomicReference<RetentionLeases> retentionLeasesHolder = new AtomicReference<>(new RetentionLeases(primaryTerm, retentionLeasesVersion.get(), Collections.emptyList()));
    final List<Engine.Operation> operations = generateSingleDocHistory(true, randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), false, 2, 10, 300, "2");
    Randomness.shuffle(operations);
    Set<Long> existingSeqNos = new HashSet<>();
    store = createStore();
    engine = createEngine(config(indexSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get, retentionLeasesHolder::get));
    assertThat(engine.getMinRetainedSeqNo(), equalTo(0L));
    long lastMinRetainedSeqNo = engine.getMinRetainedSeqNo();
    for (Engine.Operation op : operations) {
        final Engine.Result result;
        if (op instanceof Engine.Index) {
            result = engine.index((Engine.Index) op);
        } else {
            result = engine.delete((Engine.Delete) op);
        }
        existingSeqNos.add(result.getSeqNo());
        if (randomBoolean()) {
            // advance persisted local checkpoint
            engine.syncTranslog();
            assertEquals(engine.getProcessedLocalCheckpoint(), engine.getPersistedLocalCheckpoint());
            globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpointTracker().getPersistedCheckpoint()));
        }
        if (randomBoolean()) {
            retentionLeasesVersion.incrementAndGet();
            final int length = randomIntBetween(0, 8);
            final List<RetentionLease> leases = new ArrayList<>(length);
            for (int i = 0; i < length; i++) {
                final String id = randomAlphaOfLength(8);
                final long retainingSequenceNumber = randomLongBetween(0, Math.max(0, globalCheckpoint.get()));
                final long timestamp = randomLongBetween(0L, Long.MAX_VALUE);
                final String source = randomAlphaOfLength(8);
                leases.add(new RetentionLease(id, retainingSequenceNumber, timestamp, source));
            }
            retentionLeasesHolder.set(new RetentionLeases(primaryTerm, retentionLeasesVersion.get(), leases));
        }
        if (rarely()) {
            settings.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), randomLongBetween(0, 10));
            indexSettings.updateIndexMetadata(IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build());
            engine.onSettingsChanged(indexSettings.getTranslogRetentionAge(), indexSettings.getTranslogRetentionSize(), indexSettings.getSoftDeleteRetentionOperations());
        }
        if (rarely()) {
            engine.refresh("test");
        }
        if (rarely()) {
            engine.flush(true, true);
            assertThat(Long.parseLong(engine.getLastCommittedSegmentInfos().userData.get(Engine.MIN_RETAINED_SEQNO)), equalTo(engine.getMinRetainedSeqNo()));
        }
        if (rarely()) {
            engine.forceMerge(randomBoolean(), 1, false, false, false, UUIDs.randomBase64UUID());
        }
        try (Closeable ignored = engine.acquireHistoryRetentionLock(Engine.HistorySource.INDEX)) {
            long minRetainSeqNos = engine.getMinRetainedSeqNo();
            assertThat(minRetainSeqNos, lessThanOrEqualTo(globalCheckpoint.get() + 1));
            Long[] expectedOps = existingSeqNos.stream().filter(seqno -> seqno >= minRetainSeqNos).toArray(Long[]::new);
            Set<Long> actualOps = readAllOperationsInLucene(engine, createMapperService("test")).stream().map(Translog.Operation::seqNo).collect(Collectors.toSet());
            assertThat(actualOps, containsInAnyOrder(expectedOps));
        }
        try (Engine.IndexCommitRef commitRef = engine.acquireSafeIndexCommit()) {
            IndexCommit safeCommit = commitRef.getIndexCommit();
            if (safeCommit.getUserData().containsKey(Engine.MIN_RETAINED_SEQNO)) {
                lastMinRetainedSeqNo = Long.parseLong(safeCommit.getUserData().get(Engine.MIN_RETAINED_SEQNO));
            }
        }
    }
    if (randomBoolean()) {
        engine.close();
    } else {
        engine.flushAndClose();
    }
    try (InternalEngine recoveringEngine = new InternalEngine(engine.config())) {
        assertThat(recoveringEngine.getMinRetainedSeqNo(), equalTo(lastMinRetainedSeqNo));
    }
}