Examples with Translog org.elasticsearch.index.translog.Translog used on opensource projects

Example 11 with Translog

use of org.elasticsearch.index.translog.Translog in project elasticsearch by elastic.

the class RecoverySourceHandler method sendSnapshot.

/**
     * Send the given snapshot's operations with a sequence number greater than the specified staring sequence number to this handler's
     * target node.
     * <p>
     * Operations are bulked into a single request depending on an operation count limit or size-in-bytes limit.
     *
     * @param startingSeqNo the sequence number for which only operations with a sequence number greater than this will be sent
     * @param snapshot      the translog snapshot to replay operations from
     * @return the total number of translog operations that were sent
     * @throws IOException if an I/O exception occurred reading the translog snapshot
     */
protected int sendSnapshot(final long startingSeqNo, final Translog.Snapshot snapshot) throws IOException {
    int ops = 0;
    long size = 0;
    int totalOperations = 0;
    final List<Translog.Operation> operations = new ArrayList<>();
    if (snapshot.totalOperations() == 0) {
        logger.trace("no translog operations to send");
    }
    // send operations in batches
    Translog.Operation operation;
    while ((operation = snapshot.next()) != null) {
        if (shard.state() == IndexShardState.CLOSED) {
            throw new IndexShardClosedException(request.shardId());
        }
        cancellableThreads.checkForCancel();
        // if we are doing a sequence-number-based recovery, we have to skip older ops for which no sequence number was assigned, and
        // any ops before the starting sequence number
        final long seqNo = operation.seqNo();
        if (startingSeqNo >= 0 && (seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO || seqNo < startingSeqNo))
            continue;
        operations.add(operation);
        ops++;
        size += operation.estimateSize();
        totalOperations++;
        // check if this request is past bytes threshold, and if so, send it off
        if (size >= chunkSizeInBytes) {
            cancellableThreads.execute(() -> recoveryTarget.indexTranslogOperations(operations, snapshot.totalOperations()));
            if (logger.isTraceEnabled()) {
                logger.trace("sent batch of [{}][{}] (total: [{}]) translog operations", ops, new ByteSizeValue(size), snapshot.totalOperations());
            }
            ops = 0;
            size = 0;
            operations.clear();
        }
    }
    // send the leftover operations
    if (!operations.isEmpty()) {
        cancellableThreads.execute(() -> recoveryTarget.indexTranslogOperations(operations, snapshot.totalOperations()));
    }
    if (logger.isTraceEnabled()) {
        logger.trace("sent final batch of [{}][{}] (total: [{}]) translog operations", ops, new ByteSizeValue(size), snapshot.totalOperations());
    }
    return totalOperations;
}

Example 12 with Translog

use of org.elasticsearch.index.translog.Translog in project elasticsearch by elastic.

the class TranslogRecoveryPerformer method performRecoveryOperation.

/**
     * Performs a single recovery operation.
     *
     * @param allowMappingUpdates true if mapping update should be accepted (but collected). Setting it to false will
     *                            cause a {@link MapperException} to be thrown if an update
     *                            is encountered.
     */
private void performRecoveryOperation(Engine engine, Translog.Operation operation, boolean allowMappingUpdates, Engine.Operation.Origin origin) throws IOException {
    try {
        switch(operation.opType()) {
            case INDEX:
                Translog.Index index = (Translog.Index) operation;
                // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all
                // autoGeneratedID docs that are coming from the primary are updated correctly.
                Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), source(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source())).routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true);
                maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates);
                logger.trace("[translog] recover [index] op [({}, {})] of [{}][{}]", index.seqNo(), index.primaryTerm(), index.type(), index.id());
                index(engine, engineIndex);
                break;
            case DELETE:
                Translog.Delete delete = (Translog.Delete) operation;
                Uid uid = Uid.createUid(delete.uid().text());
                logger.trace("[translog] recover [delete] op [({}, {})] of [{}][{}]", delete.seqNo(), delete.primaryTerm(), uid.type(), uid.id());
                final Engine.Delete engineDelete = new Engine.Delete(uid.type(), uid.id(), delete.uid(), delete.seqNo(), delete.primaryTerm(), delete.version(), delete.versionType().versionTypeForReplicationAndRecovery(), origin, System.nanoTime());
                delete(engine, engineDelete);
                break;
            case NO_OP:
                final Translog.NoOp noOp = (Translog.NoOp) operation;
                final long seqNo = noOp.seqNo();
                final long primaryTerm = noOp.primaryTerm();
                final String reason = noOp.reason();
                logger.trace("[translog] recover [no_op] op [({}, {})] of [{}]", seqNo, primaryTerm, reason);
                final Engine.NoOp engineNoOp = new Engine.NoOp(null, seqNo, primaryTerm, 0, VersionType.INTERNAL, origin, System.nanoTime(), reason);
                noOp(engine, engineNoOp);
                break;
            default:
                throw new IllegalStateException("No operation defined for [" + operation + "]");
        }
    } catch (ElasticsearchException e) {
        boolean hasIgnoreOnRecoveryException = false;
        ElasticsearchException current = e;
        while (true) {
            if (current instanceof IgnoreOnRecoveryEngineException) {
                hasIgnoreOnRecoveryException = true;
                break;
            }
            if (current.getCause() instanceof ElasticsearchException) {
                current = (ElasticsearchException) current.getCause();
            } else {
                break;
            }
        }
        if (!hasIgnoreOnRecoveryException) {
            throw e;
        }
    }
    operationProcessed();
}

Example 13 with Translog

use of org.elasticsearch.index.translog.Translog in project elasticsearch by elastic.

the class TransportShardBulkActionTests method testUpdateReplicaRequestWithSuccess.

public void testUpdateReplicaRequestWithSuccess() throws Exception {
    DocWriteRequest writeRequest = new IndexRequest("index", "type", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value");
    BulkItemRequest replicaRequest = new BulkItemRequest(0, writeRequest);
    boolean created = randomBoolean();
    Translog.Location resultLocation = new Translog.Location(42, 42, 42);
    Engine.IndexResult indexResult = new FakeResult(1, 1, created, resultLocation);
    DocWriteResponse indexResponse = new IndexResponse(shardId, "index", "id", 1, 1, created);
    BulkItemResultHolder goodResults = new BulkItemResultHolder(indexResponse, indexResult, replicaRequest);
    Translog.Location originalLocation = new Translog.Location(21, 21, 21);
    BulkItemRequest[] items = new BulkItemRequest[0];
    BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
    Translog.Location newLocation = TransportShardBulkAction.updateReplicaRequest(goodResults, DocWriteRequest.OpType.INDEX, originalLocation, bulkShardRequest);
    BulkItemResponse primaryResponse = replicaRequest.getPrimaryResponse();
    // Check that the translog is successfully advanced
    assertThat(newLocation, equalTo(resultLocation));
    // Since this was not a conflict failure, the primary response
    // should be filled out with the failure information
    assertThat(primaryResponse.getItemId(), equalTo(0));
    assertThat(primaryResponse.getId(), equalTo("id"));
    assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
    DocWriteResponse response = primaryResponse.getResponse();
    assertThat(response.status(), equalTo(created ? RestStatus.CREATED : RestStatus.OK));
}

Example 14 with Translog

use of org.elasticsearch.index.translog.Translog in project elasticsearch by elastic.

the class TransportShardBulkActionTests method testExecuteBulkIndexRequestWithConflictingMappings.

public void testExecuteBulkIndexRequestWithConflictingMappings() throws Exception {
    IndexMetaData metaData = indexMetaData();
    IndexShard shard = newStartedShard(true);
    BulkItemRequest[] items = new BulkItemRequest[1];
    DocWriteRequest writeRequest = new IndexRequest("index", "type", "id").source(Requests.INDEX_CONTENT_TYPE, "foo", "bar");
    items[0] = new BulkItemRequest(0, writeRequest);
    BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
    Translog.Location location = new Translog.Location(0, 0, 0);
    UpdateHelper updateHelper = null;
    // Return a mapping conflict (IAE) when trying to update the mapping
    Exception err = new IllegalArgumentException("mapping conflict");
    Translog.Location newLocation = TransportShardBulkAction.executeBulkItemRequest(metaData, shard, bulkShardRequest, location, 0, updateHelper, threadPool::absoluteTimeInMillis, new FailingMappingUpdatePerformer(err));
    // Translog shouldn't change, as there were conflicting mappings
    assertThat(newLocation, equalTo(location));
    BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
    // Since this was not a conflict failure, the primary response
    // should be filled out with the failure information
    assertThat(primaryResponse.getItemId(), equalTo(0));
    assertThat(primaryResponse.getId(), equalTo("id"));
    assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
    assertTrue(primaryResponse.isFailed());
    assertThat(primaryResponse.getFailureMessage(), containsString("mapping conflict"));
    BulkItemResponse.Failure failure = primaryResponse.getFailure();
    assertThat(failure.getIndex(), equalTo("index"));
    assertThat(failure.getType(), equalTo("type"));
    assertThat(failure.getId(), equalTo("id"));
    assertThat(failure.getCause(), equalTo(err));
    assertThat(failure.getStatus(), equalTo(RestStatus.BAD_REQUEST));
    closeShards(shard);
}

Example 15 with Translog

use of org.elasticsearch.index.translog.Translog in project elasticsearch by elastic.

the class InternalEngineTests method testNoOps.

/*
     * This test tests that a no-op does not generate a new sequence number, that no-ops can advance the local checkpoint, and that no-ops
     * are correctly added to the translog.
     */
public void testNoOps() throws IOException {
    engine.close();
    InternalEngine noOpEngine = null;
    final int maxSeqNo = randomIntBetween(0, 128);
    final int localCheckpoint = randomIntBetween(0, maxSeqNo);
    final int globalCheckpoint = randomIntBetween(0, localCheckpoint);
    try {
        final SequenceNumbersService seqNoService = new SequenceNumbersService(shardId, defaultSettings, maxSeqNo, localCheckpoint, globalCheckpoint) {

            @Override
            public long generateSeqNo() {
                throw new UnsupportedOperationException();
            }
        };
        noOpEngine = createEngine(defaultSettings, store, primaryTranslogDir, newMergePolicy(), null, () -> seqNoService);
        final long primaryTerm = randomNonNegativeLong();
        final String reason = randomAsciiOfLength(16);
        noOpEngine.noOp(new Engine.NoOp(null, maxSeqNo + 1, primaryTerm, 0, VersionType.INTERNAL, randomFrom(PRIMARY, REPLICA, PEER_RECOVERY, LOCAL_TRANSLOG_RECOVERY), System.nanoTime(), reason));
        assertThat(noOpEngine.seqNoService().getLocalCheckpoint(), equalTo((long) (maxSeqNo + 1)));
        assertThat(noOpEngine.getTranslog().totalOperations(), equalTo(1));
        final Translog.Operation op = noOpEngine.getTranslog().newSnapshot().next();
        assertThat(op, instanceOf(Translog.NoOp.class));
        final Translog.NoOp noOp = (Translog.NoOp) op;
        assertThat(noOp.seqNo(), equalTo((long) (maxSeqNo + 1)));
        assertThat(noOp.primaryTerm(), equalTo(primaryTerm));
        assertThat(noOp.reason(), equalTo(reason));
    } finally {
        IOUtils.close(noOpEngine);
    }
}