Examples with RetentionLeases org.opensearch.index.seqno.RetentionLeases used on opensource projects

Example 16 with RetentionLeases

use of org.opensearch.index.seqno.RetentionLeases in project OpenSearch by opensearch-project.

the class IndexShardRetentionLeaseTests method assertRetentionLeases.

private void assertRetentionLeases(final IndexShard indexShard, final int size, final long[] minimumRetainingSequenceNumbers, final long primaryTerm, final long version, final boolean primary, final boolean expireLeases) {
    assertTrue(expireLeases == false || primary);
    final RetentionLeases retentionLeases;
    if (expireLeases == false) {
        if (randomBoolean()) {
            retentionLeases = indexShard.getRetentionLeases();
        } else {
            final Tuple<Boolean, RetentionLeases> tuple = indexShard.getRetentionLeases(false);
            assertFalse(tuple.v1());
            retentionLeases = tuple.v2();
        }
    } else {
        final Tuple<Boolean, RetentionLeases> tuple = indexShard.getRetentionLeases(true);
        assertTrue(tuple.v1());
        retentionLeases = tuple.v2();
    }
    assertRetentionLeases(retentionLeases, size, minimumRetainingSequenceNumbers, primaryTerm, version);
}

Example 17 with RetentionLeases

use of org.opensearch.index.seqno.RetentionLeases in project OpenSearch by opensearch-project.

the class IndexShardRetentionLeaseTests method runExpirationTest.

private void runExpirationTest(final boolean primary) throws IOException {
    final long retentionLeaseMillis = randomLongBetween(1, TimeValue.timeValueHours(12).millis());
    final Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING.getKey(), TimeValue.timeValueMillis(retentionLeaseMillis)).build();
    // current time is mocked through the thread pool
    final IndexShard indexShard = newStartedShard(primary, settings, new InternalEngineFactory());
    final long primaryTerm = indexShard.getOperationPrimaryTerm();
    try {
        final long[] retainingSequenceNumbers = new long[1];
        retainingSequenceNumbers[0] = randomLongBetween(0, Long.MAX_VALUE);
        final long initialVersion;
        if (primary) {
            initialVersion = 2;
            indexShard.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> {
            }));
        } else {
            initialVersion = 3;
            final RetentionLeases retentionLeases = new RetentionLeases(primaryTerm, initialVersion, Arrays.asList(peerRecoveryRetentionLease(indexShard), new RetentionLease("0", retainingSequenceNumbers[0], currentTimeMillis.get(), "test-0")));
            indexShard.updateRetentionLeasesOnReplica(retentionLeases);
        }
        {
            final RetentionLeases retentionLeases = indexShard.getEngine().config().retentionLeasesSupplier().get();
            assertThat(retentionLeases.version(), equalTo(initialVersion));
            assertThat(retentionLeases.leases(), hasSize(2));
            final RetentionLease retentionLease = retentionLeases.get("0");
            assertThat(retentionLease.timestamp(), equalTo(currentTimeMillis.get()));
            assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, initialVersion, primary, false);
        }
        // renew the lease
        currentTimeMillis.set(currentTimeMillis.get() + randomLongBetween(0, 1024));
        retainingSequenceNumbers[0] = randomLongBetween(retainingSequenceNumbers[0], Long.MAX_VALUE);
        if (primary) {
            indexShard.renewRetentionLease("0", retainingSequenceNumbers[0], "test-0");
        } else {
            final RetentionLeases retentionLeases = new RetentionLeases(primaryTerm, initialVersion + 1, Arrays.asList(peerRecoveryRetentionLease(indexShard), new RetentionLease("0", retainingSequenceNumbers[0], currentTimeMillis.get(), "test-0")));
            indexShard.updateRetentionLeasesOnReplica(retentionLeases);
        }
        {
            final RetentionLeases retentionLeases = indexShard.getEngine().config().retentionLeasesSupplier().get();
            assertThat(retentionLeases.version(), equalTo(initialVersion + 1));
            assertThat(retentionLeases.leases(), hasSize(2));
            final RetentionLease retentionLease = retentionLeases.get("0");
            assertThat(retentionLease.timestamp(), equalTo(currentTimeMillis.get()));
            assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, initialVersion + 1, primary, false);
        }
        // now force the lease to expire
        currentTimeMillis.set(currentTimeMillis.get() + randomLongBetween(retentionLeaseMillis, Long.MAX_VALUE - currentTimeMillis.get()));
        if (primary) {
            assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, initialVersion + 1, true, false);
            assertRetentionLeases(indexShard, 0, new long[0], primaryTerm, initialVersion + 2, true, true);
        } else {
            assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, initialVersion + 1, false, false);
        }
    } finally {
        closeShards(indexShard);
    }
}

Example 18 with RetentionLeases

use of org.opensearch.index.seqno.RetentionLeases in project OpenSearch by opensearch-project.

the class EngineConfigFactoryTests method testCreateEngineConfigFromFactory.

public void testCreateEngineConfigFromFactory() {
    IndexMetadata meta = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1).build();
    List<EnginePlugin> plugins = Collections.singletonList(new FooEnginePlugin());
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", meta.getSettings());
    EngineConfigFactory factory = new EngineConfigFactory(plugins, indexSettings);
    EngineConfig config = factory.newEngineConfig(null, null, indexSettings, null, null, null, null, null, null, null, null, null, null, TimeValue.timeValueMinutes(5), null, null, null, null, null, () -> new RetentionLeases(0, 0, Collections.emptyList()), null, null, false);
    assertNotNull(config.getCodec());
    assertNotNull(config.getCustomTranslogDeletionPolicyFactory());
    assertTrue(config.getCustomTranslogDeletionPolicyFactory().create(indexSettings, null) instanceof CustomTranslogDeletionPolicy);
}

Example 19 with RetentionLeases

use of org.opensearch.index.seqno.RetentionLeases in project OpenSearch by opensearch-project.

the class SoftDeletesPolicyTests method testWhenRetentionLeasesDictateThePolicy.

public void testWhenRetentionLeasesDictateThePolicy() {
    final int retentionOperations = randomIntBetween(0, 1024);
    final Collection<RetentionLease> leases = new ArrayList<>();
    final int numberOfLeases = randomIntBetween(1, 16);
    for (int i = 0; i < numberOfLeases; i++) {
        leases.add(new RetentionLease(Integer.toString(i), randomLongBetween(0, Long.MAX_VALUE - retentionOperations - 1), randomNonNegativeLong(), "test"));
    }
    final OptionalLong minimumRetainingSequenceNumber = leases.stream().mapToLong(RetentionLease::retainingSequenceNumber).min();
    assert minimumRetainingSequenceNumber.isPresent() : leases;
    final long localCheckpointOfSafeCommit = randomLongBetween(minimumRetainingSequenceNumber.getAsLong(), Long.MAX_VALUE - 1);
    final AtomicLong globalCheckpoint = new AtomicLong(randomLongBetween(minimumRetainingSequenceNumber.getAsLong() + retentionOperations, Long.MAX_VALUE - 1));
    final long primaryTerm = randomNonNegativeLong();
    final long version = randomNonNegativeLong();
    final Supplier<RetentionLeases> leasesSupplier = () -> new RetentionLeases(primaryTerm, version, Collections.unmodifiableCollection(new ArrayList<>(leases)));
    final SoftDeletesPolicy policy = new SoftDeletesPolicy(globalCheckpoint::get, 0, retentionOperations, leasesSupplier);
    policy.setLocalCheckpointOfSafeCommit(localCheckpointOfSafeCommit);
    assertThat(policy.getMinRetainedSeqNo(), equalTo(minimumRetainingSequenceNumber.getAsLong()));
}

Example 20 with RetentionLeases

use of org.opensearch.index.seqno.RetentionLeases in project OpenSearch by opensearch-project.

the class EngineTestCase method config.

public EngineConfig config(final IndexSettings indexSettings, final Store store, final Path translogPath, final MergePolicy mergePolicy, final ReferenceManager.RefreshListener externalRefreshListener, final ReferenceManager.RefreshListener internalRefreshListener, final Sort indexSort, @Nullable final LongSupplier maybeGlobalCheckpointSupplier, @Nullable final Supplier<RetentionLeases> maybeRetentionLeasesSupplier, final CircuitBreakerService breakerService) {
    final IndexWriterConfig iwc = newIndexWriterConfig();
    final TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
    final Engine.EventListener eventListener = new Engine.EventListener() {
    };
    // we don't need to notify anybody in this test
    final List<ReferenceManager.RefreshListener> extRefreshListenerList = externalRefreshListener == null ? emptyList() : Collections.singletonList(externalRefreshListener);
    final List<ReferenceManager.RefreshListener> intRefreshListenerList = internalRefreshListener == null ? emptyList() : Collections.singletonList(internalRefreshListener);
    final LongSupplier globalCheckpointSupplier;
    final Supplier<RetentionLeases> retentionLeasesSupplier;
    if (maybeGlobalCheckpointSupplier == null) {
        assert maybeRetentionLeasesSupplier == null;
        final ReplicationTracker replicationTracker = new ReplicationTracker(shardId, allocationId.getId(), indexSettings, randomNonNegativeLong(), SequenceNumbers.NO_OPS_PERFORMED, update -> {
        }, () -> 0L, (leases, listener) -> listener.onResponse(new ReplicationResponse()), () -> SafeCommitInfo.EMPTY);
        globalCheckpointSupplier = replicationTracker;
        retentionLeasesSupplier = replicationTracker::getRetentionLeases;
    } else {
        assert maybeRetentionLeasesSupplier != null;
        globalCheckpointSupplier = maybeGlobalCheckpointSupplier;
        retentionLeasesSupplier = maybeRetentionLeasesSupplier;
    }
    return new EngineConfig(shardId, threadPool, indexSettings, null, store, mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), extRefreshListenerList, intRefreshListenerList, indexSort, breakerService, globalCheckpointSupplier, retentionLeasesSupplier, primaryTerm, tombstoneDocSupplier());
}