Examples with TranslogConfig org.elasticsearch.index.translog.TranslogConfig used on opensource projects

Example 1 with TranslogConfig

use of org.elasticsearch.index.translog.TranslogConfig in project elasticsearch by elastic.

the class InternalEngineTests method testRecoverFromForeignTranslog.

public void testRecoverFromForeignTranslog() throws IOException {
    final int numDocs = randomIntBetween(1, 10);
    for (int i = 0; i < numDocs; i++) {
        ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
        Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
        Engine.IndexResult index = engine.index(firstIndexRequest);
        assertThat(index.getVersion(), equalTo(1L));
    }
    engine.refresh("test");
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
    Translog.TranslogGeneration generation = engine.getTranslog().getGeneration();
    engine.close();
    Translog translog = new Translog(new TranslogConfig(shardId, createTempDir(), INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), null, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO);
    translog.add(new Translog.Index("test", "SomeBogusId", "{}".getBytes(Charset.forName("UTF-8"))));
    assertEquals(generation.translogFileGeneration, translog.currentFileGeneration());
    translog.close();
    EngineConfig config = engine.config();
    /* create a TranslogConfig that has been created with a different UUID */
    TranslogConfig translogConfig = new TranslogConfig(shardId, translog.location(), config.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE);
    EngineConfig brokenConfig = new EngineConfig(EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, shardId, threadPool, config.getIndexSettings(), null, store, createSnapshotDeletionPolicy(), newMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), config.getRefreshListeners(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
    try {
        InternalEngine internalEngine = new InternalEngine(brokenConfig);
        fail("translog belongs to a different engine");
    } catch (EngineCreationFailureException ex) {
    }
    // and recover again!
    engine = createEngine(store, primaryTranslogDir);
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
}

Example 2 with TranslogConfig

use of org.elasticsearch.index.translog.TranslogConfig in project elasticsearch by elastic.

the class InternalEngine method openTranslog.

private Translog openTranslog(EngineConfig engineConfig, IndexWriter writer, LongSupplier globalCheckpointSupplier) throws IOException {
    assert openMode != null;
    final TranslogConfig translogConfig = engineConfig.getTranslogConfig();
    Translog.TranslogGeneration generation = null;
    if (openMode == EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG) {
        generation = loadTranslogIdFromCommit(writer);
        // We expect that this shard already exists, so it must already have an existing translog else something is badly wrong!
        if (generation == null) {
            throw new IllegalStateException("no translog generation present in commit data but translog is expected to exist");
        }
        if (generation.translogUUID == null) {
            throw new IndexFormatTooOldException("trasnlog", "translog has no generation nor a UUID - this might be an index from a previous version consider upgrading to N-1 first");
        }
    }
    final Translog translog = new Translog(translogConfig, generation, globalCheckpointSupplier);
    if (generation == null || generation.translogUUID == null) {
        assert openMode != EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG : "OpenMode must not be " + EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG;
        if (generation == null) {
            logger.debug("no translog ID present in the current generation - creating one");
        } else if (generation.translogUUID == null) {
            logger.debug("upgraded translog to pre 2.0 format, associating translog with index - writing translog UUID");
        }
        boolean success = false;
        try {
            commitIndexWriter(writer, translog, openMode == EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG ? commitDataAsMap(writer).get(SYNC_COMMIT_ID) : null);
            success = true;
        } finally {
            if (success == false) {
                IOUtils.closeWhileHandlingException(translog);
            }
        }
    }
    return translog;
}

Example 3 with TranslogConfig

use of org.elasticsearch.index.translog.TranslogConfig in project elasticsearch by elastic.

the class InternalEngineTests method config.

public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy, SnapshotDeletionPolicy deletionPolicy, long maxUnsafeAutoIdTimestamp, ReferenceManager.RefreshListener refreshListener) {
    IndexWriterConfig iwc = newIndexWriterConfig();
    TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
    final EngineConfig.OpenMode openMode;
    try {
        if (Lucene.indexExists(store.directory()) == false) {
            openMode = EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG;
        } else {
            openMode = EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG;
        }
    } catch (IOException e) {
        throw new ElasticsearchException("can't find index?", e);
    }
    Engine.EventListener listener = new Engine.EventListener() {

        @Override
        public void onFailedEngine(String reason, @Nullable Exception e) {
        // we don't need to notify anybody in this test
        }
    };
    EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings, null, store, deletionPolicy, mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), listener, new TranslogHandler(xContentRegistry(), shardId.getIndexName(), logger), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), refreshListener, maxUnsafeAutoIdTimestamp);
    return config;
}

Example 4 with TranslogConfig

use of org.elasticsearch.index.translog.TranslogConfig in project elasticsearch by elastic.

the class RefreshListenersTests method setupListeners.

@Before
public void setupListeners() throws Exception {
    // Setup dependencies of the listeners
    maxListeners = randomIntBetween(1, 1000);
    listeners = new RefreshListeners(() -> maxListeners, () -> engine.refresh("too-many-listeners"), // Immediately run listeners rather than adding them to the listener thread pool like IndexShard does to simplify the test.
    Runnable::run, logger);
    // Now setup the InternalEngine which is much more complicated because we aren't mocking anything
    threadPool = new TestThreadPool(getTestName());
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY);
    ShardId shardId = new ShardId(new Index("index", "_na_"), 1);
    Directory directory = newDirectory();
    DirectoryService directoryService = new DirectoryService(shardId, indexSettings) {

        @Override
        public Directory newDirectory() throws IOException {
            return directory;
        }
    };
    store = new Store(shardId, indexSettings, directoryService, new DummyShardLock(shardId));
    IndexWriterConfig iwc = newIndexWriterConfig();
    TranslogConfig translogConfig = new TranslogConfig(shardId, createTempDir("translog"), indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
    Engine.EventListener eventListener = new Engine.EventListener() {

        @Override
        public void onFailedEngine(String reason, @Nullable Exception e) {
        // we don't need to notify anybody in this test
        }
    };
    TranslogHandler translogHandler = new TranslogHandler(xContentRegistry(), shardId.getIndexName(), logger);
    EngineConfig config = new EngineConfig(EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG, shardId, threadPool, indexSettings, null, store, new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()), newMergePolicy(), iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, translogHandler, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), listeners, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
    engine = new InternalEngine(config);
    listeners.setTranslog(engine.getTranslog());
}

Example 5 with TranslogConfig

use of org.elasticsearch.index.translog.TranslogConfig in project elasticsearch by elastic.

the class ShadowEngineTests method config.

public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy, RefreshListeners refreshListeners) {
    IndexWriterConfig iwc = newIndexWriterConfig();
    final EngineConfig.OpenMode openMode;
    try {
        if (Lucene.indexExists(store.directory()) == false) {
            openMode = EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG;
        } else {
            openMode = EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG;
        }
    } catch (IOException e) {
        throw new ElasticsearchException("can't find index?", e);
    }
    Engine.EventListener eventListener = new Engine.EventListener() {

        @Override
        public void onFailedEngine(String reason, @Nullable Exception e) {
        // we don't need to notify anybody in this test
        }
    };
    TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
    EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings, null, store, createSnapshotDeletionPolicy(), mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, null, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), refreshListeners, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
    return config;
}