Examples with Engine org.opensearch.index.engine.Engine used on opensource projects

Example 21 with Engine

use of org.opensearch.index.engine.Engine in project OpenSearch by opensearch-project.

the class IndexShardTests method testRefreshListenersDuringPeerRecovery.

public void testRefreshListenersDuringPeerRecovery() throws IOException {
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetadata metadata = IndexMetadata.builder("test").putMapping("{ \"properties\": { \"foo\":  { \"type\": \"text\"}}}").settings(settings).primaryTerm(0, 1).build();
    IndexShard primary = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, null);
    recoverShardFromStore(primary);
    indexDoc(primary, "_doc", "0", "{\"foo\" : \"bar\"}");
    Consumer<IndexShard> assertListenerCalled = shard -> {
        AtomicBoolean called = new AtomicBoolean();
        shard.addRefreshListener(null, b -> {
            assertFalse(b);
            called.set(true);
        });
        assertTrue(called.get());
    };
    IndexShard replica = newShard(primary.shardId(), false, "n2", metadata, null);
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    replica.markAsRecovering("for testing", new RecoveryState(replica.routingEntry(), localNode, localNode));
    assertListenerCalled.accept(replica);
    recoverReplica(replica, primary, (shard, discoveryNode) -> new RecoveryTarget(shard, discoveryNode, recoveryListener) {

        // we're only checking that listeners are called when the engine is open, before there is no point
        @Override
        public void prepareForTranslogOperations(int totalTranslogOps, ActionListener<Void> listener) {
            super.prepareForTranslogOperations(totalTranslogOps, ActionListener.wrap(r -> {
                assertListenerCalled.accept(replica);
                listener.onResponse(r);
            }, listener::onFailure));
        }

        @Override
        public void indexTranslogOperations(final List<Translog.Operation> operations, final int totalTranslogOps, final long maxAutoIdTimestamp, final long maxSeqNoOfUpdatesOrDeletes, final RetentionLeases retentionLeases, final long mappingVersion, final ActionListener<Long> listener) {
            super.indexTranslogOperations(operations, totalTranslogOps, maxAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, mappingVersion, ActionListener.wrap(r -> {
                assertListenerCalled.accept(replica);
                listener.onResponse(r);
            }, listener::onFailure));
        }

        @Override
        public void finalizeRecovery(long globalCheckpoint, long trimAboveSeqNo, ActionListener<Void> listener) {
            super.finalizeRecovery(globalCheckpoint, trimAboveSeqNo, ActionListener.wrap(r -> {
                assertListenerCalled.accept(replica);
                listener.onResponse(r);
            }, listener::onFailure));
        }
    }, false, true);
    closeShards(primary, replica);
}

Example 22 with Engine

use of org.opensearch.index.engine.Engine in project OpenSearch by opensearch-project.

the class IndexShardTests method testRollbackReplicaEngineOnPromotion.

public void testRollbackReplicaEngineOnPromotion() throws IOException, InterruptedException {
    final IndexShard indexShard = newStartedShard(false);
    // most of the time this is large enough that most of the time there will be at least one gap
    final int operations = 1024 - scaledRandomIntBetween(0, 1024);
    indexOnReplicaWithGaps(indexShard, operations, Math.toIntExact(SequenceNumbers.NO_OPS_PERFORMED));
    final long globalCheckpointOnReplica = randomLongBetween(UNASSIGNED_SEQ_NO, indexShard.getLocalCheckpoint());
    indexShard.updateGlobalCheckpointOnReplica(globalCheckpointOnReplica, "test");
    final long globalCheckpoint = randomLongBetween(UNASSIGNED_SEQ_NO, indexShard.getLocalCheckpoint());
    Set<String> docsBelowGlobalCheckpoint = getShardDocUIDs(indexShard).stream().filter(id -> Long.parseLong(id) <= Math.max(globalCheckpointOnReplica, globalCheckpoint)).collect(Collectors.toSet());
    final CountDownLatch latch = new CountDownLatch(1);
    final boolean shouldRollback = Math.max(globalCheckpoint, globalCheckpointOnReplica) < indexShard.seqNoStats().getMaxSeqNo() && indexShard.seqNoStats().getMaxSeqNo() != SequenceNumbers.NO_OPS_PERFORMED;
    final Engine beforeRollbackEngine = indexShard.getEngine();
    final long newMaxSeqNoOfUpdates = randomLongBetween(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), Long.MAX_VALUE);
    randomReplicaOperationPermitAcquisition(indexShard, indexShard.getPendingPrimaryTerm() + 1, globalCheckpoint, newMaxSeqNoOfUpdates, new ActionListener<Releasable>() {

        @Override
        public void onResponse(final Releasable releasable) {
            releasable.close();
            latch.countDown();
        }

        @Override
        public void onFailure(final Exception e) {
        }
    }, "");
    latch.await();
    if (globalCheckpointOnReplica == UNASSIGNED_SEQ_NO && globalCheckpoint == UNASSIGNED_SEQ_NO) {
        assertThat(indexShard.getLocalCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
    } else {
        assertThat(indexShard.getLocalCheckpoint(), equalTo(Math.max(globalCheckpoint, globalCheckpointOnReplica)));
    }
    assertThat(getShardDocUIDs(indexShard), equalTo(docsBelowGlobalCheckpoint));
    if (shouldRollback) {
        assertThat(indexShard.getEngine(), not(sameInstance(beforeRollbackEngine)));
    } else {
        assertThat(indexShard.getEngine(), sameInstance(beforeRollbackEngine));
    }
    assertThat(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(newMaxSeqNoOfUpdates));
    // ensure that after the local checkpoint throw back and indexing again, the local checkpoint advances
    final Result result = indexOnReplicaWithGaps(indexShard, operations, Math.toIntExact(indexShard.getLocalCheckpoint()));
    assertThat(indexShard.getLocalCheckpoint(), equalTo((long) result.localCheckpoint));
    closeShard(indexShard, false);
}

Example 23 with Engine

use of org.opensearch.index.engine.Engine in project OpenSearch by opensearch-project.

the class IndexServiceTests method testAsyncTranslogTrimTaskOnClosedIndex.

public void testAsyncTranslogTrimTaskOnClosedIndex() throws Exception {
    final String indexName = "test";
    IndexService indexService = createIndex(indexName, Settings.builder().put(TRANSLOG_RETENTION_CHECK_INTERVAL_SETTING.getKey(), "100ms").build());
    Translog translog = IndexShardTestCase.getTranslog(indexService.getShard(0));
    int translogOps = 0;
    final int numDocs = scaledRandomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        client().prepareIndex().setIndex(indexName).setId(String.valueOf(i)).setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
        translogOps++;
        if (randomBoolean()) {
            client().admin().indices().prepareFlush(indexName).get();
            if (indexService.getIndexSettings().isSoftDeleteEnabled()) {
                translogOps = 0;
            }
        }
    }
    assertThat(translog.totalOperations(), equalTo(translogOps));
    assertThat(translog.stats().estimatedNumberOfOperations(), equalTo(translogOps));
    assertAcked(client().admin().indices().prepareClose("test").setWaitForActiveShards(ActiveShardCount.DEFAULT));
    indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(indexService.index());
    assertTrue(indexService.getTrimTranslogTask().mustReschedule());
    final Engine readOnlyEngine = getEngine(indexService.getShard(0));
    assertBusy(() -> assertThat(readOnlyEngine.getTranslogStats().getTranslogSizeInBytes(), equalTo((long) Translog.DEFAULT_HEADER_SIZE_IN_BYTES)));
    assertAcked(client().admin().indices().prepareOpen("test").setWaitForActiveShards(ActiveShardCount.DEFAULT));
    indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(indexService.index());
    translog = IndexShardTestCase.getTranslog(indexService.getShard(0));
    assertThat(translog.totalOperations(), equalTo(0));
    assertThat(translog.stats().estimatedNumberOfOperations(), equalTo(0));
}

Example 24 with Engine

use of org.opensearch.index.engine.Engine in project OpenSearch by opensearch-project.

the class IndexShard method close.

public void close(String reason, boolean flushEngine) throws IOException {
    synchronized (engineMutex) {
        try {
            synchronized (mutex) {
                changeState(IndexShardState.CLOSED, reason);
            }
        } finally {
            final Engine engine = this.currentEngineReference.getAndSet(null);
            try {
                if (engine != null && flushEngine) {
                    engine.flushAndClose();
                }
            } finally {
                // playing safe here and close the engine even if the above succeeds - close can be called multiple times
                // Also closing refreshListeners to prevent us from accumulating any more listeners
                IOUtils.close(engine, globalCheckpointListeners, refreshListeners, pendingReplicationActions);
                indexShardOperationPermits.close();
            }
        }
    }
}

Example 25 with Engine

use of org.opensearch.index.engine.Engine in project OpenSearch by opensearch-project.

the class IndexShard method onSettingsChanged.

public void onSettingsChanged() {
    Engine engineOrNull = getEngineOrNull();
    if (engineOrNull != null) {
        final boolean disableTranslogRetention = indexSettings.isSoftDeleteEnabled() && useRetentionLeasesInPeerRecovery;
        engineOrNull.onSettingsChanged(disableTranslogRetention ? TimeValue.MINUS_ONE : indexSettings.getTranslogRetentionAge(), disableTranslogRetention ? new ByteSizeValue(-1) : indexSettings.getTranslogRetentionSize(), indexSettings.getSoftDeleteRetentionOperations());
    }
}