Examples with Translog org.elasticsearch.index.translog.Translog used on opensource projects

Example 1 with Translog

use of org.elasticsearch.index.translog.Translog in project crate by crate.

the class BlobRecoverySourceHandler method recoverToTarget.

/**
     * performs the recovery from the local engine to the target
     */
public RecoveryResponse recoverToTarget() {
    final Engine engine = shard.engine();
    assert engine.getTranslog() != null : "translog must not be null";
    try (Translog.View translogView = engine.getTranslog().newView()) {
        logger.trace("captured translog id [{}] for recovery", translogView.minTranslogGeneration());
        final SnapshotIndexCommit phase1Snapshot;
        try {
            phase1Snapshot = shard.snapshotIndex(false);
        } catch (Throwable e) {
            IOUtils.closeWhileHandlingException(translogView);
            throw new RecoveryEngineException(shard.shardId(), 1, "Snapshot failed", e);
        }
        try {
            phase1(phase1Snapshot, translogView);
        } catch (Throwable e) {
            throw new RecoveryEngineException(shard.shardId(), 1, "phase1 failed", e);
        } finally {
            Releasables.closeWhileHandlingException(phase1Snapshot);
        }
        logger.trace("snapshot translog for recovery. current size is [{}]", translogView.totalOperations());
        try (Translog.Snapshot phase2Snapshot = translogView.snapshot()) {
            phase2(phase2Snapshot);
        } catch (Throwable e) {
            throw new RecoveryEngineException(shard.shardId(), 2, "phase2 failed", e);
        }
        finalizeRecovery();
    }
    return response;
}

Example 2 with Translog

use of org.elasticsearch.index.translog.Translog in project crate by crate.

the class BlobRecoverySourceHandler method sendSnapshot.

/**
     * Send the given snapshot's operations to this handler's target node.
     * <p/>
     * Operations are bulked into a single request depending on an operation
     * count limit or size-in-bytes limit
     *
     * @return the total number of translog operations that were sent
     */
protected int sendSnapshot(final Translog.Snapshot snapshot) {
    int ops = 0;
    long size = 0;
    int totalOperations = 0;
    final List<Translog.Operation> operations = new ArrayList<>();
    Translog.Operation operation;
    try {
        // this ex should bubble up
        operation = snapshot.next();
    } catch (IOException ex) {
        throw new ElasticsearchException("failed to get next operation from translog", ex);
    }
    final TransportRequestOptions recoveryOptions = TransportRequestOptions.builder().withCompress(recoverySettings.compress()).withType(TransportRequestOptions.Type.RECOVERY).withTimeout(recoverySettings.internalActionLongTimeout()).build();
    if (operation == null) {
        logger.trace("[{}][{}] no translog operations to send to {}", indexName, shardId, request.targetNode());
    }
    while (operation != null) {
        if (shard.state() == IndexShardState.CLOSED) {
            throw new IndexShardClosedException(request.shardId());
        }
        cancellableThreads.checkForCancel();
        operations.add(operation);
        ops += 1;
        size += operation.estimateSize();
        totalOperations++;
        // if so, send it off
        if (ops >= recoverySettings.translogOps() || size >= recoverySettings.translogSize().getBytes()) {
            // don't throttle translog, since we lock for phase3 indexing,
            // so we need to move it as fast as possible. Note, since we
            // index docs to replicas while the index files are recovered
            // the lock can potentially be removed, in which case, it might
            // make sense to re-enable throttling in this phase
            //                if (recoverySettings.rateLimiter() != null) {
            //                    recoverySettings.rateLimiter().pause(size);
            //                }
            cancellableThreads.execute(new Interruptable() {

                @Override
                public void run() throws InterruptedException {
                    final RecoveryTranslogOperationsRequest translogOperationsRequest = new RecoveryTranslogOperationsRequest(request.recoveryId(), request.shardId(), operations, snapshot.estimatedTotalOperations());
                    transportService.submitRequest(request.targetNode(), RecoveryTarget.Actions.TRANSLOG_OPS, translogOperationsRequest, recoveryOptions, EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
                }
            });
            if (logger.isTraceEnabled()) {
                logger.trace("[{}][{}] sent batch of [{}][{}] (total: [{}]) translog operations to {}", indexName, shardId, ops, new ByteSizeValue(size), snapshot.estimatedTotalOperations(), request.targetNode());
            }
            ops = 0;
            size = 0;
            operations.clear();
        }
        try {
            // this ex should bubble up
            operation = snapshot.next();
        } catch (IOException ex) {
            throw new ElasticsearchException("failed to get next operation from translog", ex);
        }
    }
    // send the leftover
    if (!operations.isEmpty()) {
        cancellableThreads.execute(new Interruptable() {

            @Override
            public void run() throws InterruptedException {
                RecoveryTranslogOperationsRequest translogOperationsRequest = new RecoveryTranslogOperationsRequest(request.recoveryId(), request.shardId(), operations, snapshot.estimatedTotalOperations());
                transportService.submitRequest(request.targetNode(), RecoveryTarget.Actions.TRANSLOG_OPS, translogOperationsRequest, recoveryOptions, EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
            }
        });
    }
    if (logger.isTraceEnabled()) {
        logger.trace("[{}][{}] sent final batch of [{}][{}] (total: [{}]) translog operations to {}", indexName, shardId, ops, new ByteSizeValue(size), snapshot.estimatedTotalOperations(), request.targetNode());
    }
    return totalOperations;
}

Example 3 with Translog

use of org.elasticsearch.index.translog.Translog in project elasticsearch by elastic.

the class InternalEngineTests method testRecoverFromForeignTranslog.

public void testRecoverFromForeignTranslog() throws IOException {
    final int numDocs = randomIntBetween(1, 10);
    for (int i = 0; i < numDocs; i++) {
        ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
        Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
        Engine.IndexResult index = engine.index(firstIndexRequest);
        assertThat(index.getVersion(), equalTo(1L));
    }
    engine.refresh("test");
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
    Translog.TranslogGeneration generation = engine.getTranslog().getGeneration();
    engine.close();
    Translog translog = new Translog(new TranslogConfig(shardId, createTempDir(), INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE), null, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO);
    translog.add(new Translog.Index("test", "SomeBogusId", "{}".getBytes(Charset.forName("UTF-8"))));
    assertEquals(generation.translogFileGeneration, translog.currentFileGeneration());
    translog.close();
    EngineConfig config = engine.config();
    /* create a TranslogConfig that has been created with a different UUID */
    TranslogConfig translogConfig = new TranslogConfig(shardId, translog.location(), config.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE);
    EngineConfig brokenConfig = new EngineConfig(EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, shardId, threadPool, config.getIndexSettings(), null, store, createSnapshotDeletionPolicy(), newMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), config.getRefreshListeners(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
    try {
        InternalEngine internalEngine = new InternalEngine(brokenConfig);
        fail("translog belongs to a different engine");
    } catch (EngineCreationFailureException ex) {
    }
    // and recover again!
    engine = createEngine(store, primaryTranslogDir);
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
}

Example 4 with Translog

use of org.elasticsearch.index.translog.Translog in project elasticsearch by elastic.

the class InternalEngineTests method testMissingTranslog.

public void testMissingTranslog() throws IOException {
    // test that we can force start the engine , even if the translog is missing.
    engine.close();
    // fake a new translog, causing the engine to point to a missing one.
    Translog translog = createTranslog();
    long id = translog.currentFileGeneration();
    translog.close();
    IOUtils.rm(translog.location().resolve(Translog.getFilename(id)));
    try {
        engine = createEngine(store, primaryTranslogDir);
        fail("engine shouldn't start without a valid translog id");
    } catch (EngineCreationFailureException ex) {
    // expected
    }
    // now it should be OK.
    EngineConfig config = copy(config(defaultSettings, store, primaryTranslogDir, newMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null), EngineConfig.OpenMode.OPEN_INDEX_CREATE_TRANSLOG);
    engine = new InternalEngine(config);
}

Example 5 with Translog

use of org.elasticsearch.index.translog.Translog in project elasticsearch by elastic.

the class GlobalCheckpointSyncActionTests method testTranslogSyncAfterGlobalCheckpointSync.

public void testTranslogSyncAfterGlobalCheckpointSync() throws Exception {
    final IndicesService indicesService = mock(IndicesService.class);
    final Index index = new Index("index", "uuid");
    final IndexService indexService = mock(IndexService.class);
    when(indicesService.indexServiceSafe(index)).thenReturn(indexService);
    final int id = randomIntBetween(0, 4);
    final IndexShard indexShard = mock(IndexShard.class);
    when(indexService.getShard(id)).thenReturn(indexShard);
    final Translog translog = mock(Translog.class);
    when(indexShard.getTranslog()).thenReturn(translog);
    final GlobalCheckpointSyncAction action = new GlobalCheckpointSyncAction(Settings.EMPTY, transportService, clusterService, indicesService, threadPool, shardStateAction, new ActionFilters(Collections.emptySet()), new IndexNameExpressionResolver(Settings.EMPTY));
    final ShardId shardId = new ShardId(index, id);
    final GlobalCheckpointSyncAction.PrimaryRequest primaryRequest = new GlobalCheckpointSyncAction.PrimaryRequest(shardId);
    if (randomBoolean()) {
        action.shardOperationOnPrimary(primaryRequest, indexShard);
    } else {
        action.shardOperationOnReplica(new GlobalCheckpointSyncAction.ReplicaRequest(primaryRequest, randomNonNegativeLong()), indexShard);
    }
    verify(translog).sync();
}