Examples with InternalEngineFactory org.opensearch.index.engine.InternalEngineFactory used on opensource projects

Example 6 with InternalEngineFactory

use of org.opensearch.index.engine.InternalEngineFactory in project OpenSearch by opensearch-project.

the class IndexShardRetentionLeaseTests method testPersistence.

public void testPersistence() throws IOException {
    final Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING.getKey(), Long.MAX_VALUE, TimeUnit.NANOSECONDS).build();
    final IndexShard indexShard = newStartedShard(true, settings, new InternalEngineFactory());
    try {
        final int length = randomIntBetween(0, 8);
        final long[] minimumRetainingSequenceNumbers = new long[length];
        for (int i = 0; i < length; i++) {
            minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
            currentTimeMillis.set(TimeUnit.NANOSECONDS.toMillis(randomNonNegativeLong()));
            indexShard.addRetentionLease(Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> {
            }));
        }
        currentTimeMillis.set(TimeUnit.NANOSECONDS.toMillis(Long.MAX_VALUE));
        // force the retention leases to persist
        indexShard.persistRetentionLeases();
        // the written retention leases should equal our current retention leases
        final RetentionLeases retentionLeases = indexShard.getEngine().config().retentionLeasesSupplier().get();
        final RetentionLeases writtenRetentionLeases = indexShard.loadRetentionLeases();
        assertThat(writtenRetentionLeases.version(), equalTo(1L + length));
        assertThat(writtenRetentionLeases.leases(), contains(retentionLeases.leases().toArray(new RetentionLease[0])));
        // when we recover, we should recover the retention leases
        final IndexShard recoveredShard = reinitShard(indexShard, ShardRoutingHelper.initWithSameId(indexShard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE));
        try {
            recoverShardFromStore(recoveredShard);
            final RetentionLeases recoveredRetentionLeases = recoveredShard.getEngine().config().retentionLeasesSupplier().get();
            assertThat(recoveredRetentionLeases.version(), equalTo(1L + length));
            assertThat(recoveredRetentionLeases.leases(), contains(retentionLeases.leases().toArray(new RetentionLease[0])));
        } finally {
            closeShards(recoveredShard);
        }
        // we should not recover retention leases when force-allocating a stale primary
        final IndexShard forceRecoveredShard = reinitShard(indexShard, ShardRoutingHelper.initWithSameId(indexShard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE));
        try {
            recoverShardFromStore(forceRecoveredShard);
            final RetentionLeases recoveredRetentionLeases = forceRecoveredShard.getEngine().config().retentionLeasesSupplier().get();
            assertThat(recoveredRetentionLeases.leases(), hasSize(1));
            assertThat(recoveredRetentionLeases.leases().iterator().next().id(), equalTo(ReplicationTracker.getPeerRecoveryRetentionLeaseId(indexShard.routingEntry())));
            assertThat(recoveredRetentionLeases.version(), equalTo(1L));
        } finally {
            closeShards(forceRecoveredShard);
        }
    } finally {
        closeShards(indexShard);
    }
}

Example 7 with InternalEngineFactory

use of org.opensearch.index.engine.InternalEngineFactory in project OpenSearch by opensearch-project.

the class IndexShardTests method testClosedIndicesSkipSyncGlobalCheckpoint.

public void testClosedIndicesSkipSyncGlobalCheckpoint() throws Exception {
    ShardId shardId = new ShardId("index", "_na_", 0);
    IndexMetadata.Builder indexMetadata = IndexMetadata.builder("index").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2)).state(IndexMetadata.State.CLOSE).primaryTerm(0, 1);
    ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(8), true, ShardRoutingState.INITIALIZING, RecoverySource.EmptyStoreRecoverySource.INSTANCE);
    AtomicBoolean synced = new AtomicBoolean();
    IndexShard primaryShard = newShard(shardRouting, indexMetadata.build(), null, new InternalEngineFactory(), () -> synced.set(true), RetentionLeaseSyncer.EMPTY);
    recoverShardFromStore(primaryShard);
    IndexShard replicaShard = newShard(shardId, false);
    recoverReplica(replicaShard, primaryShard, true);
    int numDocs = between(1, 10);
    for (int i = 0; i < numDocs; i++) {
        indexDoc(primaryShard, "_doc", Integer.toString(i));
    }
    assertThat(primaryShard.getLocalCheckpoint(), equalTo(numDocs - 1L));
    primaryShard.updateLocalCheckpointForShard(replicaShard.shardRouting.allocationId().getId(), primaryShard.getLocalCheckpoint());
    long globalCheckpointOnReplica = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, primaryShard.getLocalCheckpoint());
    primaryShard.updateGlobalCheckpointForShard(replicaShard.shardRouting.allocationId().getId(), globalCheckpointOnReplica);
    primaryShard.maybeSyncGlobalCheckpoint("test");
    assertFalse("closed indices should skip global checkpoint sync", synced.get());
    closeShards(primaryShard, replicaShard);
}

Example 8 with InternalEngineFactory

use of org.opensearch.index.engine.InternalEngineFactory in project OpenSearch by opensearch-project.

the class IndexShardTests method testDoNotTrimCommitsWhenOpenReadOnlyEngine.

public void testDoNotTrimCommitsWhenOpenReadOnlyEngine() throws Exception {
    final IndexShard shard = newStartedShard(false, Settings.EMPTY, new InternalEngineFactory());
    long numDocs = randomLongBetween(1, 20);
    long seqNo = 0;
    for (long i = 0; i < numDocs; i++) {
        if (rarely()) {
            // create gaps in sequence numbers
            seqNo++;
        }
        shard.applyIndexOperationOnReplica(seqNo, shard.getOperationPrimaryTerm(), 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, new SourceToParse(shard.shardId.getIndexName(), Long.toString(i), new BytesArray("{}"), XContentType.JSON));
        shard.updateGlobalCheckpointOnReplica(shard.getLocalCheckpoint(), "test");
        if (randomInt(100) < 10) {
            shard.flush(new FlushRequest());
        }
        seqNo++;
    }
    shard.flush(new FlushRequest());
    assertThat(shard.docStats().getCount(), equalTo(numDocs));
    final ShardRouting replicaRouting = shard.routingEntry();
    ShardRouting readonlyShardRouting = newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, RecoverySource.ExistingStoreRecoverySource.INSTANCE);
    final IndexShard readonlyShard = reinitShard(shard, readonlyShardRouting, shard.indexSettings.getIndexMetadata(), engineConfig -> new ReadOnlyEngine(engineConfig, null, null, true, Function.identity(), true) {

        @Override
        protected void ensureMaxSeqNoEqualsToGlobalCheckpoint(SeqNoStats seqNoStats) {
        // just like a following shard, we need to skip this check for now.
        }
    }, shard.getEngineConfigFactory());
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    readonlyShard.markAsRecovering("store", new RecoveryState(readonlyShard.routingEntry(), localNode, null));
    recoverFromStore(readonlyShard);
    assertThat(readonlyShard.docStats().getCount(), equalTo(numDocs));
    closeShards(readonlyShard);
}

Example 9 with InternalEngineFactory

use of org.opensearch.index.engine.InternalEngineFactory in project OpenSearch by opensearch-project.

the class IndexShardTests method testGlobalCheckpointSync.

public void testGlobalCheckpointSync() throws IOException {
    // create the primary shard with a callback that sets a boolean when the global checkpoint sync is invoked
    final ShardId shardId = new ShardId("index", "_na_", 0);
    final ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(8), true, ShardRoutingState.INITIALIZING, RecoverySource.EmptyStoreRecoverySource.INSTANCE);
    final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    final IndexMetadata.Builder indexMetadata = IndexMetadata.builder(shardRouting.getIndexName()).settings(settings).primaryTerm(0, 1);
    final AtomicBoolean synced = new AtomicBoolean();
    final IndexShard primaryShard = newShard(shardRouting, indexMetadata.build(), null, new InternalEngineFactory(), () -> synced.set(true), RetentionLeaseSyncer.EMPTY);
    // add a replica
    recoverShardFromStore(primaryShard);
    final IndexShard replicaShard = newShard(shardId, false);
    recoverReplica(replicaShard, primaryShard, true);
    final int maxSeqNo = randomIntBetween(0, 128);
    for (int i = 0; i <= maxSeqNo; i++) {
        EngineTestCase.generateNewSeqNo(primaryShard.getEngine());
    }
    final long checkpoint = rarely() ? maxSeqNo - scaledRandomIntBetween(0, maxSeqNo) : maxSeqNo;
    // set up local checkpoints on the shard copies
    primaryShard.updateLocalCheckpointForShard(shardRouting.allocationId().getId(), checkpoint);
    final int replicaLocalCheckpoint = randomIntBetween(0, Math.toIntExact(checkpoint));
    final String replicaAllocationId = replicaShard.routingEntry().allocationId().getId();
    primaryShard.updateLocalCheckpointForShard(replicaAllocationId, replicaLocalCheckpoint);
    // initialize the local knowledge on the primary of the persisted global checkpoint on the replica shard
    final int replicaGlobalCheckpoint = randomIntBetween(Math.toIntExact(SequenceNumbers.NO_OPS_PERFORMED), Math.toIntExact(primaryShard.getLastKnownGlobalCheckpoint()));
    primaryShard.updateGlobalCheckpointForShard(replicaAllocationId, replicaGlobalCheckpoint);
    // initialize the local knowledge on the primary of the persisted global checkpoint on the primary
    primaryShard.updateGlobalCheckpointForShard(shardRouting.allocationId().getId(), primaryShard.getLastKnownGlobalCheckpoint());
    // simulate a background maybe sync; it should only run if the knowledge on the replica of the global checkpoint lags the primary
    primaryShard.maybeSyncGlobalCheckpoint("test");
    assertThat(synced.get(), equalTo(maxSeqNo == primaryShard.getLastKnownGlobalCheckpoint() && (replicaGlobalCheckpoint < checkpoint)));
    // simulate that the background sync advanced the global checkpoint on the replica
    primaryShard.updateGlobalCheckpointForShard(replicaAllocationId, primaryShard.getLastKnownGlobalCheckpoint());
    // reset our boolean so that we can assert after another simulated maybe sync
    synced.set(false);
    primaryShard.maybeSyncGlobalCheckpoint("test");
    // this time there should not be a sync since all the replica copies are caught up with the primary
    assertFalse(synced.get());
    closeShards(replicaShard, primaryShard);
}

Example 10 with InternalEngineFactory

use of org.opensearch.index.engine.InternalEngineFactory in project OpenSearch by opensearch-project.

the class RecoveryDuringReplicationTests method testDoNotWaitForPendingSeqNo.

public void testDoNotWaitForPendingSeqNo() throws Exception {
    IndexMetadata metadata = buildIndexMetadata(1);
    final int pendingDocs = randomIntBetween(1, 5);
    final BlockingEngineFactory primaryEngineFactory = new BlockingEngineFactory();
    try (ReplicationGroup shards = new ReplicationGroup(metadata) {

        @Override
        protected EngineFactory getEngineFactory(ShardRouting routing) {
            if (routing.primary()) {
                return primaryEngineFactory;
            } else {
                return new InternalEngineFactory();
            }
        }
    }) {
        shards.startAll();
        int docs = shards.indexDocs(randomIntBetween(1, 10));
        // simulate a background global checkpoint sync at which point we expect the global checkpoint to advance on the replicas
        shards.syncGlobalCheckpoint();
        IndexShard replica = shards.getReplicas().get(0);
        shards.removeReplica(replica);
        closeShards(replica);
        docs += pendingDocs;
        primaryEngineFactory.latchIndexers(pendingDocs);
        CountDownLatch pendingDocsDone = new CountDownLatch(pendingDocs);
        for (int i = 0; i < pendingDocs; i++) {
            final String id = "pending_" + i;
            threadPool.generic().submit(() -> {
                try {
                    shards.index(new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON));
                } catch (Exception e) {
                    throw new AssertionError(e);
                } finally {
                    pendingDocsDone.countDown();
                }
            });
        }
        // wait for the pending ops to "hang"
        primaryEngineFactory.awaitIndexersLatch();
        primaryEngineFactory.allowIndexing();
        // index some more
        docs += shards.indexDocs(randomInt(5));
        IndexShard newReplica = shards.addReplicaWithExistingPath(replica.shardPath(), replica.routingEntry().currentNodeId());
        CountDownLatch recoveryStart = new CountDownLatch(1);
        AtomicBoolean recoveryDone = new AtomicBoolean(false);
        final Future<Void> recoveryFuture = shards.asyncRecoverReplica(newReplica, (indexShard, node) -> {
            recoveryStart.countDown();
            return new RecoveryTarget(indexShard, node, recoveryListener) {

                @Override
                public void finalizeRecovery(long globalCheckpoint, long trimAboveSeqNo, ActionListener<Void> listener) {
                    recoveryDone.set(true);
                    super.finalizeRecovery(globalCheckpoint, trimAboveSeqNo, listener);
                }
            };
        });
        recoveryStart.await();
        // index some more
        final int indexedDuringRecovery = shards.indexDocs(randomInt(5));
        docs += indexedDuringRecovery;
        assertBusy(() -> assertTrue("recovery should not wait for on pending docs", recoveryDone.get()));
        primaryEngineFactory.releaseLatchedIndexers();
        pendingDocsDone.await();
        // now recovery can finish
        recoveryFuture.get();
        assertThat(newReplica.recoveryState().getIndex().fileDetails(), empty());
        shards.assertAllEqual(docs);
    } finally {
        primaryEngineFactory.close();
    }
}