Examples with Engine org.opensearch.index.engine.Engine used on opensource projects

Example 31 with Engine

use of org.opensearch.index.engine.Engine in project OpenSearch by opensearch-project.

the class IndexShardTests method testRollbackReplicaEngineOnPromotion.

public void testRollbackReplicaEngineOnPromotion() throws IOException, InterruptedException {
    final IndexShard indexShard = newStartedShard(false);
    // most of the time this is large enough that most of the time there will be at least one gap
    final int operations = 1024 - scaledRandomIntBetween(0, 1024);
    indexOnReplicaWithGaps(indexShard, operations, Math.toIntExact(SequenceNumbers.NO_OPS_PERFORMED));
    final long globalCheckpointOnReplica = randomLongBetween(UNASSIGNED_SEQ_NO, indexShard.getLocalCheckpoint());
    indexShard.updateGlobalCheckpointOnReplica(globalCheckpointOnReplica, "test");
    final long globalCheckpoint = randomLongBetween(UNASSIGNED_SEQ_NO, indexShard.getLocalCheckpoint());
    Set<String> docsBelowGlobalCheckpoint = getShardDocUIDs(indexShard).stream().filter(id -> Long.parseLong(id) <= Math.max(globalCheckpointOnReplica, globalCheckpoint)).collect(Collectors.toSet());
    final CountDownLatch latch = new CountDownLatch(1);
    final boolean shouldRollback = Math.max(globalCheckpoint, globalCheckpointOnReplica) < indexShard.seqNoStats().getMaxSeqNo() && indexShard.seqNoStats().getMaxSeqNo() != SequenceNumbers.NO_OPS_PERFORMED;
    final Engine beforeRollbackEngine = indexShard.getEngine();
    final long newMaxSeqNoOfUpdates = randomLongBetween(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), Long.MAX_VALUE);
    randomReplicaOperationPermitAcquisition(indexShard, indexShard.getPendingPrimaryTerm() + 1, globalCheckpoint, newMaxSeqNoOfUpdates, new ActionListener<Releasable>() {

        @Override
        public void onResponse(final Releasable releasable) {
            releasable.close();
            latch.countDown();
        }

        @Override
        public void onFailure(final Exception e) {
        }
    }, "");
    latch.await();
    if (globalCheckpointOnReplica == UNASSIGNED_SEQ_NO && globalCheckpoint == UNASSIGNED_SEQ_NO) {
        assertThat(indexShard.getLocalCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
    } else {
        assertThat(indexShard.getLocalCheckpoint(), equalTo(Math.max(globalCheckpoint, globalCheckpointOnReplica)));
    }
    assertThat(getShardDocUIDs(indexShard), equalTo(docsBelowGlobalCheckpoint));
    if (shouldRollback) {
        assertThat(indexShard.getEngine(), not(sameInstance(beforeRollbackEngine)));
    } else {
        assertThat(indexShard.getEngine(), sameInstance(beforeRollbackEngine));
    }
    assertThat(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(newMaxSeqNoOfUpdates));
    // ensure that after the local checkpoint throw back and indexing again, the local checkpoint advances
    final Result result = indexOnReplicaWithGaps(indexShard, operations, Math.toIntExact(indexShard.getLocalCheckpoint()));
    assertThat(indexShard.getLocalCheckpoint(), equalTo((long) result.localCheckpoint));
    closeShard(indexShard, false);
}

Example 32 with Engine

use of org.opensearch.index.engine.Engine in project OpenSearch by opensearch-project.

the class IndexShardTests method testCloseShardWhileResettingEngine.

/**
 * This test simulates a scenario seen rarely in ConcurrentSeqNoVersioningIT. Closing a shard while engine is inside
 * resetEngineToGlobalCheckpoint can lead to check index failure in integration tests.
 */
public void testCloseShardWhileResettingEngine() throws Exception {
    CountDownLatch readyToCloseLatch = new CountDownLatch(1);
    CountDownLatch closeDoneLatch = new CountDownLatch(1);
    IndexShard shard = newStartedShard(false, Settings.EMPTY, config -> new InternalEngine(config) {

        @Override
        public InternalEngine recoverFromTranslog(TranslogRecoveryRunner translogRecoveryRunner, long recoverUpToSeqNo) throws IOException {
            readyToCloseLatch.countDown();
            try {
                closeDoneLatch.await();
            } catch (InterruptedException e) {
                throw new AssertionError(e);
            }
            return super.recoverFromTranslog(translogRecoveryRunner, recoverUpToSeqNo);
        }
    });
    Thread closeShardThread = new Thread(() -> {
        try {
            readyToCloseLatch.await();
            shard.close("testing", false);
            // in integration tests, this is done as a listener on IndexService.
            MockFSDirectoryFactory.checkIndex(logger, shard.store(), shard.shardId);
        } catch (InterruptedException | IOException e) {
            throw new AssertionError(e);
        } finally {
            closeDoneLatch.countDown();
        }
    });
    closeShardThread.start();
    final CountDownLatch engineResetLatch = new CountDownLatch(1);
    shard.acquireAllReplicaOperationsPermits(shard.getOperationPrimaryTerm(), shard.getLastKnownGlobalCheckpoint(), 0L, ActionListener.wrap(r -> {
        try (Releasable dummy = r) {
            shard.resetEngineToGlobalCheckpoint();
        } finally {
            engineResetLatch.countDown();
        }
    }, Assert::assertNotNull), TimeValue.timeValueMinutes(1L));
    engineResetLatch.await();
    closeShardThread.join();
    // close store.
    closeShard(shard, false);
}

Example 33 with Engine

use of org.opensearch.index.engine.Engine in project OpenSearch by opensearch-project.

the class IndexServiceTests method testAsyncTranslogTrimTaskOnClosedIndex.

public void testAsyncTranslogTrimTaskOnClosedIndex() throws Exception {
    final String indexName = "test";
    IndexService indexService = createIndex(indexName, Settings.builder().put(TRANSLOG_RETENTION_CHECK_INTERVAL_SETTING.getKey(), "100ms").build());
    Translog translog = IndexShardTestCase.getTranslog(indexService.getShard(0));
    int translogOps = 0;
    final int numDocs = scaledRandomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        client().prepareIndex().setIndex(indexName).setId(String.valueOf(i)).setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
        translogOps++;
        if (randomBoolean()) {
            client().admin().indices().prepareFlush(indexName).get();
            if (indexService.getIndexSettings().isSoftDeleteEnabled()) {
                translogOps = 0;
            }
        }
    }
    assertThat(translog.totalOperations(), equalTo(translogOps));
    assertThat(translog.stats().estimatedNumberOfOperations(), equalTo(translogOps));
    assertAcked(client().admin().indices().prepareClose("test").setWaitForActiveShards(ActiveShardCount.DEFAULT));
    indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(indexService.index());
    assertTrue(indexService.getTrimTranslogTask().mustReschedule());
    final Engine readOnlyEngine = getEngine(indexService.getShard(0));
    assertBusy(() -> assertThat(readOnlyEngine.getTranslogStats().getTranslogSizeInBytes(), equalTo((long) Translog.DEFAULT_HEADER_SIZE_IN_BYTES)));
    assertAcked(client().admin().indices().prepareOpen("test").setWaitForActiveShards(ActiveShardCount.DEFAULT));
    indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(indexService.index());
    translog = IndexShardTestCase.getTranslog(indexService.getShard(0));
    assertThat(translog.totalOperations(), equalTo(0));
    assertThat(translog.stats().estimatedNumberOfOperations(), equalTo(0));
}

Example 34 with Engine

use of org.opensearch.index.engine.Engine in project OpenSearch by opensearch-project.

the class IndexShardTests method testSnapshotWhileResettingEngine.

/**
 * This test simulates a scenario seen rarely in ConcurrentSeqNoVersioningIT. While engine is inside
 * resetEngineToGlobalCheckpoint snapshot metadata could fail
 */
public void testSnapshotWhileResettingEngine() throws Exception {
    CountDownLatch readyToSnapshotLatch = new CountDownLatch(1);
    CountDownLatch snapshotDoneLatch = new CountDownLatch(1);
    IndexShard shard = newStartedShard(false, Settings.EMPTY, config -> new InternalEngine(config) {

        @Override
        public InternalEngine recoverFromTranslog(TranslogRecoveryRunner translogRecoveryRunner, long recoverUpToSeqNo) throws IOException {
            InternalEngine internalEngine = super.recoverFromTranslog(translogRecoveryRunner, recoverUpToSeqNo);
            readyToSnapshotLatch.countDown();
            try {
                snapshotDoneLatch.await();
            } catch (InterruptedException e) {
                throw new AssertionError(e);
            }
            return internalEngine;
        }
    });
    indexOnReplicaWithGaps(shard, between(0, 1000), Math.toIntExact(shard.getLocalCheckpoint()));
    final long globalCheckpoint = randomLongBetween(shard.getLastKnownGlobalCheckpoint(), shard.getLocalCheckpoint());
    shard.updateGlobalCheckpointOnReplica(globalCheckpoint, "test");
    Thread snapshotThread = new Thread(() -> {
        try {
            readyToSnapshotLatch.await();
            shard.snapshotStoreMetadata();
            try (Engine.IndexCommitRef indexCommitRef = shard.acquireLastIndexCommit(randomBoolean())) {
                shard.store().getMetadata(indexCommitRef.getIndexCommit());
            }
            try (Engine.IndexCommitRef indexCommitRef = shard.acquireSafeIndexCommit()) {
                shard.store().getMetadata(indexCommitRef.getIndexCommit());
            }
        } catch (InterruptedException | IOException e) {
            throw new AssertionError(e);
        } finally {
            snapshotDoneLatch.countDown();
        }
    });
    snapshotThread.start();
    final CountDownLatch engineResetLatch = new CountDownLatch(1);
    shard.acquireAllReplicaOperationsPermits(shard.getOperationPrimaryTerm(), shard.getLastKnownGlobalCheckpoint(), 0L, ActionListener.wrap(r -> {
        try (Releasable dummy = r) {
            shard.resetEngineToGlobalCheckpoint();
        } finally {
            engineResetLatch.countDown();
        }
    }, Assert::assertNotNull), TimeValue.timeValueMinutes(1L));
    engineResetLatch.await();
    snapshotThread.join();
    closeShard(shard, false);
}