Examples with InternalEngineFactory org.opensearch.index.engine.InternalEngineFactory used on opensource projects

Example 1 with InternalEngineFactory

use of org.opensearch.index.engine.InternalEngineFactory in project OpenSearch by opensearch-project.

the class RecoveryDuringReplicationTests method testDoNotWaitForPendingSeqNo.

public void testDoNotWaitForPendingSeqNo() throws Exception {
    IndexMetadata metadata = buildIndexMetadata(1);
    final int pendingDocs = randomIntBetween(1, 5);
    final BlockingEngineFactory primaryEngineFactory = new BlockingEngineFactory();
    try (ReplicationGroup shards = new ReplicationGroup(metadata) {

        @Override
        protected EngineFactory getEngineFactory(ShardRouting routing) {
            if (routing.primary()) {
                return primaryEngineFactory;
            } else {
                return new InternalEngineFactory();
            }
        }
    }) {
        shards.startAll();
        int docs = shards.indexDocs(randomIntBetween(1, 10));
        // simulate a background global checkpoint sync at which point we expect the global checkpoint to advance on the replicas
        shards.syncGlobalCheckpoint();
        IndexShard replica = shards.getReplicas().get(0);
        shards.removeReplica(replica);
        closeShards(replica);
        docs += pendingDocs;
        primaryEngineFactory.latchIndexers(pendingDocs);
        CountDownLatch pendingDocsDone = new CountDownLatch(pendingDocs);
        for (int i = 0; i < pendingDocs; i++) {
            final String id = "pending_" + i;
            threadPool.generic().submit(() -> {
                try {
                    shards.index(new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON));
                } catch (Exception e) {
                    throw new AssertionError(e);
                } finally {
                    pendingDocsDone.countDown();
                }
            });
        }
        // wait for the pending ops to "hang"
        primaryEngineFactory.awaitIndexersLatch();
        primaryEngineFactory.allowIndexing();
        // index some more
        docs += shards.indexDocs(randomInt(5));
        IndexShard newReplica = shards.addReplicaWithExistingPath(replica.shardPath(), replica.routingEntry().currentNodeId());
        CountDownLatch recoveryStart = new CountDownLatch(1);
        AtomicBoolean recoveryDone = new AtomicBoolean(false);
        final Future<Void> recoveryFuture = shards.asyncRecoverReplica(newReplica, (indexShard, node) -> {
            recoveryStart.countDown();
            return new RecoveryTarget(indexShard, node, recoveryListener) {

                @Override
                public void finalizeRecovery(long globalCheckpoint, long trimAboveSeqNo, ActionListener<Void> listener) {
                    recoveryDone.set(true);
                    super.finalizeRecovery(globalCheckpoint, trimAboveSeqNo, listener);
                }
            };
        });
        recoveryStart.await();
        // index some more
        final int indexedDuringRecovery = shards.indexDocs(randomInt(5));
        docs += indexedDuringRecovery;
        assertBusy(() -> assertTrue("recovery should not wait for on pending docs", recoveryDone.get()));
        primaryEngineFactory.releaseLatchedIndexers();
        pendingDocsDone.await();
        // now recovery can finish
        recoveryFuture.get();
        assertThat(newReplica.recoveryState().getIndex().fileDetails(), empty());
        shards.assertAllEqual(docs);
    } finally {
        primaryEngineFactory.close();
    }
}

Example 2 with InternalEngineFactory

use of org.opensearch.index.engine.InternalEngineFactory in project OpenSearch by opensearch-project.

the class IndexShardTests method testClosedIndicesSkipSyncGlobalCheckpoint.

public void testClosedIndicesSkipSyncGlobalCheckpoint() throws Exception {
    ShardId shardId = new ShardId("index", "_na_", 0);
    IndexMetadata.Builder indexMetadata = IndexMetadata.builder("index").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2)).state(IndexMetadata.State.CLOSE).primaryTerm(0, 1);
    ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(8), true, ShardRoutingState.INITIALIZING, RecoverySource.EmptyStoreRecoverySource.INSTANCE);
    AtomicBoolean synced = new AtomicBoolean();
    IndexShard primaryShard = newShard(shardRouting, indexMetadata.build(), null, new InternalEngineFactory(), () -> synced.set(true), RetentionLeaseSyncer.EMPTY);
    recoverShardFromStore(primaryShard);
    IndexShard replicaShard = newShard(shardId, false);
    recoverReplica(replicaShard, primaryShard, true);
    int numDocs = between(1, 10);
    for (int i = 0; i < numDocs; i++) {
        indexDoc(primaryShard, "_doc", Integer.toString(i));
    }
    assertThat(primaryShard.getLocalCheckpoint(), equalTo(numDocs - 1L));
    primaryShard.updateLocalCheckpointForShard(replicaShard.shardRouting.allocationId().getId(), primaryShard.getLocalCheckpoint());
    long globalCheckpointOnReplica = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, primaryShard.getLocalCheckpoint());
    primaryShard.updateGlobalCheckpointForShard(replicaShard.shardRouting.allocationId().getId(), globalCheckpointOnReplica);
    primaryShard.maybeSyncGlobalCheckpoint("test");
    assertFalse("closed indices should skip global checkpoint sync", synced.get());
    closeShards(primaryShard, replicaShard);
}

Example 3 with InternalEngineFactory

use of org.opensearch.index.engine.InternalEngineFactory in project OpenSearch by opensearch-project.

the class IndexShardTests method testDoNotTrimCommitsWhenOpenReadOnlyEngine.

public void testDoNotTrimCommitsWhenOpenReadOnlyEngine() throws Exception {
    final IndexShard shard = newStartedShard(false, Settings.EMPTY, new InternalEngineFactory());
    long numDocs = randomLongBetween(1, 20);
    long seqNo = 0;
    for (long i = 0; i < numDocs; i++) {
        if (rarely()) {
            // create gaps in sequence numbers
            seqNo++;
        }
        shard.applyIndexOperationOnReplica(seqNo, shard.getOperationPrimaryTerm(), 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, new SourceToParse(shard.shardId.getIndexName(), "_doc", Long.toString(i), new BytesArray("{}"), XContentType.JSON));
        shard.updateGlobalCheckpointOnReplica(shard.getLocalCheckpoint(), "test");
        if (randomInt(100) < 10) {
            shard.flush(new FlushRequest());
        }
        seqNo++;
    }
    shard.flush(new FlushRequest());
    assertThat(shard.docStats().getCount(), equalTo(numDocs));
    final ShardRouting replicaRouting = shard.routingEntry();
    ShardRouting readonlyShardRouting = newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, RecoverySource.ExistingStoreRecoverySource.INSTANCE);
    final IndexShard readonlyShard = reinitShard(shard, readonlyShardRouting, shard.indexSettings.getIndexMetadata(), engineConfig -> new ReadOnlyEngine(engineConfig, null, null, true, Function.identity(), true) {

        @Override
        protected void ensureMaxSeqNoEqualsToGlobalCheckpoint(SeqNoStats seqNoStats) {
        // just like a following shard, we need to skip this check for now.
        }
    }, shard.getEngineConfigFactory());
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    readonlyShard.markAsRecovering("store", new RecoveryState(readonlyShard.routingEntry(), localNode, null));
    recoverFromStore(readonlyShard);
    assertThat(readonlyShard.docStats().getCount(), equalTo(numDocs));
    closeShards(readonlyShard);
}

Example 4 with InternalEngineFactory

use of org.opensearch.index.engine.InternalEngineFactory in project OpenSearch by opensearch-project.

the class RemoveCorruptedShardDataCommandTests method setup.

@Before
public void setup() throws IOException {
    shardId = new ShardId("index0", UUIDs.randomBase64UUID(), 0);
    final String nodeId = randomAlphaOfLength(10);
    routing = TestShardRouting.newShardRouting(shardId, nodeId, true, ShardRoutingState.INITIALIZING, RecoverySource.EmptyStoreRecoverySource.INSTANCE);
    final Path dataDir = createTempDir();
    environment = TestEnvironment.newEnvironment(Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), dataDir).putList(Environment.PATH_DATA_SETTING.getKey(), dataDir.toAbsolutePath().toString()).build());
    // create same directory structure as prod does
    final Path path = NodeEnvironment.resolveNodePath(dataDir, 0);
    Files.createDirectories(path);
    dataPaths = new Path[] { path };
    final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(MergePolicyConfig.INDEX_MERGE_ENABLED, false).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_INDEX_UUID, shardId.getIndex().getUUID()).build();
    final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(path);
    shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
    final IndexMetadata.Builder metadata = IndexMetadata.builder(routing.getIndexName()).settings(settings).primaryTerm(0, randomIntBetween(1, 100)).putMapping("_doc", "{ \"properties\": {} }");
    indexMetadata = metadata.build();
    clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(indexMetadata, false).build()).build();
    try (NodeEnvironment.NodeLock lock = new NodeEnvironment.NodeLock(0, logger, environment, Files::exists)) {
        final Path[] dataPaths = Arrays.stream(lock.getNodePaths()).filter(Objects::nonNull).map(p -> p.path).toArray(Path[]::new);
        try (PersistedClusterStateService.Writer writer = new PersistedClusterStateService(dataPaths, nodeId, xContentRegistry(), BigArrays.NON_RECYCLING_INSTANCE, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L).createWriter()) {
            writer.writeFullStateAndCommit(1L, clusterState);
        }
    }
    indexShard = newStartedShard(p -> newShard(routing, shardPath, indexMetadata, null, null, new InternalEngineFactory(), new EngineConfigFactory(new IndexSettings(indexMetadata, settings)), () -> {
    }, RetentionLeaseSyncer.EMPTY, EMPTY_EVENT_LISTENER), true);
    translogPath = shardPath.resolveTranslog();
    indexPath = shardPath.resolveIndex();
}

Example 5 with InternalEngineFactory

use of org.opensearch.index.engine.InternalEngineFactory in project OpenSearch by opensearch-project.

the class IndexShardTests method testGlobalCheckpointSync.

public void testGlobalCheckpointSync() throws IOException {
    // create the primary shard with a callback that sets a boolean when the global checkpoint sync is invoked
    final ShardId shardId = new ShardId("index", "_na_", 0);
    final ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(8), true, ShardRoutingState.INITIALIZING, RecoverySource.EmptyStoreRecoverySource.INSTANCE);
    final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    final IndexMetadata.Builder indexMetadata = IndexMetadata.builder(shardRouting.getIndexName()).settings(settings).primaryTerm(0, 1);
    final AtomicBoolean synced = new AtomicBoolean();
    final IndexShard primaryShard = newShard(shardRouting, indexMetadata.build(), null, new InternalEngineFactory(), () -> synced.set(true), RetentionLeaseSyncer.EMPTY);
    // add a replica
    recoverShardFromStore(primaryShard);
    final IndexShard replicaShard = newShard(shardId, false);
    recoverReplica(replicaShard, primaryShard, true);
    final int maxSeqNo = randomIntBetween(0, 128);
    for (int i = 0; i <= maxSeqNo; i++) {
        EngineTestCase.generateNewSeqNo(primaryShard.getEngine());
    }
    final long checkpoint = rarely() ? maxSeqNo - scaledRandomIntBetween(0, maxSeqNo) : maxSeqNo;
    // set up local checkpoints on the shard copies
    primaryShard.updateLocalCheckpointForShard(shardRouting.allocationId().getId(), checkpoint);
    final int replicaLocalCheckpoint = randomIntBetween(0, Math.toIntExact(checkpoint));
    final String replicaAllocationId = replicaShard.routingEntry().allocationId().getId();
    primaryShard.updateLocalCheckpointForShard(replicaAllocationId, replicaLocalCheckpoint);
    // initialize the local knowledge on the primary of the persisted global checkpoint on the replica shard
    final int replicaGlobalCheckpoint = randomIntBetween(Math.toIntExact(SequenceNumbers.NO_OPS_PERFORMED), Math.toIntExact(primaryShard.getLastKnownGlobalCheckpoint()));
    primaryShard.updateGlobalCheckpointForShard(replicaAllocationId, replicaGlobalCheckpoint);
    // initialize the local knowledge on the primary of the persisted global checkpoint on the primary
    primaryShard.updateGlobalCheckpointForShard(shardRouting.allocationId().getId(), primaryShard.getLastKnownGlobalCheckpoint());
    // simulate a background maybe sync; it should only run if the knowledge on the replica of the global checkpoint lags the primary
    primaryShard.maybeSyncGlobalCheckpoint("test");
    assertThat(synced.get(), equalTo(maxSeqNo == primaryShard.getLastKnownGlobalCheckpoint() && (replicaGlobalCheckpoint < checkpoint)));
    // simulate that the background sync advanced the global checkpoint on the replica
    primaryShard.updateGlobalCheckpointForShard(replicaAllocationId, primaryShard.getLastKnownGlobalCheckpoint());
    // reset our boolean so that we can assert after another simulated maybe sync
    synced.set(false);
    primaryShard.maybeSyncGlobalCheckpoint("test");
    // this time there should not be a sync since all the replica copies are caught up with the primary
    assertFalse(synced.get());
    closeShards(replicaShard, primaryShard);
}