Examples with EngineConfigFactory org.opensearch.index.engine.EngineConfigFactory used on opensource projects

Example 6 with EngineConfigFactory

use of org.opensearch.index.engine.EngineConfigFactory in project OpenSearch by opensearch-project.

the class BlobStoreRepositoryRestoreTests method testRestoreSnapshotWithExistingFiles.

/**
 * Restoring a snapshot that contains multiple files must succeed even when
 * some files already exist in the shard's store.
 */
public void testRestoreSnapshotWithExistingFiles() throws IOException {
    final IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
    final ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 0);
    IndexShard shard = newShard(shardId, true);
    try {
        // index documents in the shards
        final int numDocs = scaledRandomIntBetween(1, 500);
        recoverShardFromStore(shard);
        for (int i = 0; i < numDocs; i++) {
            indexDoc(shard, "_doc", Integer.toString(i));
            if (rarely()) {
                flushShard(shard, false);
            }
        }
        assertDocCount(shard, numDocs);
        // snapshot the shard
        final Repository repository = createRepository();
        final Snapshot snapshot = new Snapshot(repository.getMetadata().name(), new SnapshotId(randomAlphaOfLength(10), "_uuid"));
        snapshotShard(shard, snapshot, repository);
        // capture current store files
        final Store.MetadataSnapshot storeFiles = shard.snapshotStoreMetadata();
        assertFalse(storeFiles.asMap().isEmpty());
        // close the shard
        closeShards(shard);
        // delete some random files in the store
        List<String> deletedFiles = randomSubsetOf(randomIntBetween(1, storeFiles.size() - 1), storeFiles.asMap().keySet());
        for (String deletedFile : deletedFiles) {
            Files.delete(shard.shardPath().resolveIndex().resolve(deletedFile));
        }
        // build a new shard using the same store directory as the closed shard
        ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE);
        shard = newShard(shardRouting, shard.shardPath(), shard.indexSettings().getIndexMetadata(), null, null, new InternalEngineFactory(), new EngineConfigFactory(shard.indexSettings()), () -> {
        }, RetentionLeaseSyncer.EMPTY, EMPTY_EVENT_LISTENER);
        // restore the shard
        recoverShardFromSnapshot(shard, snapshot, repository);
        // check that the shard is not corrupted
        TestUtil.checkIndex(shard.store().directory());
        // check that all files have been restored
        final Directory directory = shard.store().directory();
        final List<String> directoryFiles = Arrays.asList(directory.listAll());
        for (StoreFileMetadata storeFile : storeFiles) {
            String fileName = storeFile.name();
            assertTrue("File [" + fileName + "] does not exist in store directory", directoryFiles.contains(fileName));
            assertEquals(storeFile.length(), shard.store().directory().fileLength(fileName));
        }
    } finally {
        if (shard != null && shard.state() != IndexShardState.CLOSED) {
            try {
                shard.close("test", false);
            } finally {
                IOUtils.close(shard.store());
            }
        }
    }
}

Example 7 with EngineConfigFactory

use of org.opensearch.index.engine.EngineConfigFactory in project OpenSearch by opensearch-project.

the class IndexShardTestCase method newShard.

/**
 * creates a new initializing shard. The shard will will be put in its proper path under the
 * current node id the shard is assigned to.
 * @param routing                shard routing to use
 * @param indexMetadata          indexMetadata for the shard, including any mapping
 * @param indexReaderWrapper     an optional wrapper to be used during search
 * @param globalCheckpointSyncer callback for syncing global checkpoints
 * @param listeners              an optional set of listeners to add to the shard
 */
protected IndexShard newShard(ShardRouting routing, IndexMetadata indexMetadata, @Nullable CheckedFunction<DirectoryReader, DirectoryReader, IOException> indexReaderWrapper, @Nullable EngineFactory engineFactory, Runnable globalCheckpointSyncer, RetentionLeaseSyncer retentionLeaseSyncer, IndexingOperationListener... listeners) throws IOException {
    // add node id as name to settings for proper logging
    final ShardId shardId = routing.shardId();
    final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir());
    ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
    return newShard(routing, shardPath, indexMetadata, null, indexReaderWrapper, engineFactory, new EngineConfigFactory(new IndexSettings(indexMetadata, indexMetadata.getSettings())), globalCheckpointSyncer, retentionLeaseSyncer, EMPTY_EVENT_LISTENER, listeners);
}

Example 8 with EngineConfigFactory

use of org.opensearch.index.engine.EngineConfigFactory in project OpenSearch by opensearch-project.

the class IndexShardTests method testShardStatsWithFailures.

public void testShardStatsWithFailures() throws IOException {
    allowShardFailures();
    final ShardId shardId = new ShardId("index", "_na_", 0);
    final ShardRouting shardRouting = newShardRouting(shardId, "node", true, ShardRoutingState.INITIALIZING, RecoverySource.EmptyStoreRecoverySource.INSTANCE);
    final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir());
    ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetadata metadata = IndexMetadata.builder(shardRouting.getIndexName()).settings(settings).primaryTerm(0, 1).build();
    // Override two Directory methods to make them fail at our will
    // We use AtomicReference here to inject failure in the middle of the test not immediately
    // We use Supplier<IOException> instead of IOException to produce meaningful stacktrace
    // (remember stack trace is filled when exception is instantiated)
    AtomicReference<Supplier<IOException>> exceptionToThrow = new AtomicReference<>();
    AtomicBoolean throwWhenMarkingStoreCorrupted = new AtomicBoolean(false);
    Directory directory = new FilterDirectory(newFSDirectory(shardPath.resolveIndex())) {

        // fileLength method is called during storeStats try block
        // it's not called when store is marked as corrupted
        @Override
        public long fileLength(String name) throws IOException {
            Supplier<IOException> ex = exceptionToThrow.get();
            if (ex == null) {
                return super.fileLength(name);
            } else {
                throw ex.get();
            }
        }

        // listAll method is called when marking store as corrupted
        @Override
        public String[] listAll() throws IOException {
            Supplier<IOException> ex = exceptionToThrow.get();
            if (throwWhenMarkingStoreCorrupted.get() && ex != null) {
                throw ex.get();
            } else {
                return super.listAll();
            }
        }

        // temporary override until LUCENE-8735 is integrated
        @Override
        public Set<String> getPendingDeletions() throws IOException {
            return in.getPendingDeletions();
        }
    };
    try (Store store = createStore(shardId, new IndexSettings(metadata, Settings.EMPTY), directory)) {
        IndexShard shard = newShard(shardRouting, shardPath, metadata, i -> store, null, new InternalEngineFactory(), new EngineConfigFactory(new IndexSettings(metadata, metadata.getSettings())), () -> {
        }, RetentionLeaseSyncer.EMPTY, EMPTY_EVENT_LISTENER);
        AtomicBoolean failureCallbackTriggered = new AtomicBoolean(false);
        shard.addShardFailureCallback((ig) -> failureCallbackTriggered.set(true));
        recoverShardFromStore(shard);
        final boolean corruptIndexException = randomBoolean();
        if (corruptIndexException) {
            exceptionToThrow.set(() -> new CorruptIndexException("Test CorruptIndexException", "Test resource"));
            throwWhenMarkingStoreCorrupted.set(randomBoolean());
        } else {
            exceptionToThrow.set(() -> new IOException("Test IOException"));
        }
        OpenSearchException e = expectThrows(OpenSearchException.class, shard::storeStats);
        assertTrue(failureCallbackTriggered.get());
        if (corruptIndexException && !throwWhenMarkingStoreCorrupted.get()) {
            assertTrue(store.isMarkedCorrupted());
        }
    }
}

Example 9 with EngineConfigFactory

use of org.opensearch.index.engine.EngineConfigFactory in project OpenSearch by opensearch-project.

the class IndexModuleTests method testWrapperIsBound.

public void testWrapperIsBound() throws IOException {
    final MockEngineFactory engineFactory = new MockEngineFactory(AssertingDirectoryReader.class);
    IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, engineFactory, new EngineConfigFactory(indexSettings), Collections.emptyMap(), () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), Collections.emptyMap());
    module.setReaderWrapper(s -> new Wrapper());
    IndexService indexService = newIndexService(module);
    assertTrue(indexService.getReaderWrapper() instanceof Wrapper);
    assertSame(indexService.getEngineFactory(), module.getEngineFactory());
    indexService.close("simon says", false);
}

Example 10 with EngineConfigFactory

use of org.opensearch.index.engine.EngineConfigFactory in project OpenSearch by opensearch-project.

the class PeerRecoveryTargetServiceTests method testClosedIndexSkipsLocalRecovery.

public void testClosedIndexSkipsLocalRecovery() throws Exception {
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
    IndexShard shard = newStartedShard(false);
    long globalCheckpoint = populateRandomData(shard).getGlobalCheckpoint();
    Optional<SequenceNumbers.CommitInfo> safeCommit = shard.store().findSafeIndexCommit(globalCheckpoint);
    assertTrue(safeCommit.isPresent());
    final IndexMetadata indexMetadata;
    if (randomBoolean()) {
        indexMetadata = IndexMetadata.builder(shard.indexSettings().getIndexMetadata()).settings(shard.indexSettings().getSettings()).state(IndexMetadata.State.CLOSE).build();
    } else {
        indexMetadata = IndexMetadata.builder(shard.indexSettings().getIndexMetadata()).settings(Settings.builder().put(shard.indexSettings().getSettings()).put(IndexMetadata.SETTING_BLOCKS_WRITE, true)).build();
    }
    IndexShard replica = reinitShard(shard, ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.PeerRecoverySource.INSTANCE), indexMetadata, NoOpEngine::new, new EngineConfigFactory(shard.indexSettings()));
    replica.markAsRecovering("for testing", new RecoveryState(replica.routingEntry(), localNode, localNode));
    replica.prepareForIndexRecovery();
    assertThat(replica.recoverLocallyUpToGlobalCheckpoint(), equalTo(safeCommit.get().localCheckpoint + 1));
    assertThat(replica.recoveryState().getTranslog().totalLocal(), equalTo(0));
    assertThat(replica.recoveryState().getTranslog().recoveredOperations(), equalTo(0));
    assertThat(replica.getLastKnownGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
    closeShards(replica);
}