Examples with DocWriteRequest org.elasticsearch.action.DocWriteRequest used on opensource projects

Example 1 with DocWriteRequest

use of org.elasticsearch.action.DocWriteRequest in project elasticsearch by elastic.

the class Request method bulk.

static Request bulk(BulkRequest bulkRequest) throws IOException {
    Params parameters = Params.builder();
    parameters.withTimeout(bulkRequest.timeout());
    parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
    // Bulk API only supports newline delimited JSON or Smile. Before executing
    // the bulk, we need to check that all requests have the same content-type
    // and this content-type is supported by the Bulk API.
    XContentType bulkContentType = null;
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> request = bulkRequest.requests().get(i);
        DocWriteRequest.OpType opType = request.opType();
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            bulkContentType = enforceSameContentType((IndexRequest) request, bulkContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) request;
            if (updateRequest.doc() != null) {
                bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
            }
            if (updateRequest.upsertRequest() != null) {
                bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
            }
        }
    }
    if (bulkContentType == null) {
        bulkContentType = XContentType.JSON;
    }
    byte separator = bulkContentType.xContent().streamSeparator();
    ContentType requestContentType = ContentType.create(bulkContentType.mediaType());
    ByteArrayOutputStream content = new ByteArrayOutputStream();
    for (DocWriteRequest<?> request : bulkRequest.requests()) {
        DocWriteRequest.OpType opType = request.opType();
        try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
            metadata.startObject();
            {
                metadata.startObject(opType.getLowercase());
                if (Strings.hasLength(request.index())) {
                    metadata.field("_index", request.index());
                }
                if (Strings.hasLength(request.type())) {
                    metadata.field("_type", request.type());
                }
                if (Strings.hasLength(request.id())) {
                    metadata.field("_id", request.id());
                }
                if (Strings.hasLength(request.routing())) {
                    metadata.field("_routing", request.routing());
                }
                if (Strings.hasLength(request.parent())) {
                    metadata.field("_parent", request.parent());
                }
                if (request.version() != Versions.MATCH_ANY) {
                    metadata.field("_version", request.version());
                }
                VersionType versionType = request.versionType();
                if (versionType != VersionType.INTERNAL) {
                    if (versionType == VersionType.EXTERNAL) {
                        metadata.field("_version_type", "external");
                    } else if (versionType == VersionType.EXTERNAL_GTE) {
                        metadata.field("_version_type", "external_gte");
                    } else if (versionType == VersionType.FORCE) {
                        metadata.field("_version_type", "force");
                    }
                }
                if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
                    IndexRequest indexRequest = (IndexRequest) request;
                    if (Strings.hasLength(indexRequest.getPipeline())) {
                        metadata.field("pipeline", indexRequest.getPipeline());
                    }
                } else if (opType == DocWriteRequest.OpType.UPDATE) {
                    UpdateRequest updateRequest = (UpdateRequest) request;
                    if (updateRequest.retryOnConflict() > 0) {
                        metadata.field("_retry_on_conflict", updateRequest.retryOnConflict());
                    }
                    if (updateRequest.fetchSource() != null) {
                        metadata.field("_source", updateRequest.fetchSource());
                    }
                }
                metadata.endObject();
            }
            metadata.endObject();
            BytesRef metadataSource = metadata.bytes().toBytesRef();
            content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
            content.write(separator);
        }
        BytesRef source = null;
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest indexRequest = (IndexRequest) request;
            BytesReference indexSource = indexRequest.source();
            XContentType indexXContentType = indexRequest.getContentType();
            try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, indexSource, indexXContentType)) {
                try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                    builder.copyCurrentStructure(parser);
                    source = builder.bytes().toBytesRef();
                }
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            source = XContentHelper.toXContent((UpdateRequest) request, bulkContentType, false).toBytesRef();
        }
        if (source != null) {
            content.write(source.bytes, source.offset, source.length);
            content.write(separator);
        }
    }
    HttpEntity entity = new ByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType);
    return new Request(HttpPost.METHOD_NAME, "/_bulk", parameters.getParams(), entity);
}

Example 2 with DocWriteRequest

use of org.elasticsearch.action.DocWriteRequest in project elasticsearch by elastic.

the class BulkRequest method writeTo.

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    waitForActiveShards.writeTo(out);
    out.writeVInt(requests.size());
    for (DocWriteRequest request : requests) {
        DocWriteRequest.writeDocumentRequest(out, request);
    }
    refreshPolicy.writeTo(out);
    timeout.writeTo(out);
}

Example 3 with DocWriteRequest

use of org.elasticsearch.action.DocWriteRequest in project elasticsearch by elastic.

the class TransportBulkAction method doExecute.

@Override
protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener<BulkResponse> listener) {
    if (bulkRequest.hasIndexRequestsWithPipelines()) {
        if (clusterService.localNode().isIngestNode()) {
            processBulkIndexIngestRequest(task, bulkRequest, listener);
        } else {
            ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, listener);
        }
        return;
    }
    final long startTime = relativeTime();
    final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
    if (needToCheck()) {
        // Keep track of all unique indices and all unique types per index for the create index requests:
        final Set<String> autoCreateIndices = bulkRequest.requests.stream().map(DocWriteRequest::index).collect(Collectors.toSet());
        final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size());
        ClusterState state = clusterService.state();
        for (String index : autoCreateIndices) {
            if (shouldAutoCreate(index, state)) {
                CreateIndexRequest createIndexRequest = new CreateIndexRequest();
                createIndexRequest.index(index);
                createIndexRequest.cause("auto(bulk api)");
                createIndexRequest.masterNodeTimeout(bulkRequest.timeout());
                createIndexAction.execute(createIndexRequest, new ActionListener<CreateIndexResponse>() {

                    @Override
                    public void onResponse(CreateIndexResponse result) {
                        if (counter.decrementAndGet() == 0) {
                            executeBulk(task, bulkRequest, startTime, listener, responses);
                        }
                    }

                    @Override
                    public void onFailure(Exception e) {
                        if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) {
                            // fail all requests involving this index, if create didnt work
                            for (int i = 0; i < bulkRequest.requests.size(); i++) {
                                DocWriteRequest request = bulkRequest.requests.get(i);
                                if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
                                    bulkRequest.requests.set(i, null);
                                }
                            }
                        }
                        if (counter.decrementAndGet() == 0) {
                            executeBulk(task, bulkRequest, startTime, ActionListener.wrap(listener::onResponse, inner -> {
                                inner.addSuppressed(e);
                                listener.onFailure(inner);
                            }), responses);
                        }
                    }
                });
            } else {
                if (counter.decrementAndGet() == 0) {
                    executeBulk(task, bulkRequest, startTime, listener, responses);
                }
            }
        }
    } else {
        executeBulk(task, bulkRequest, startTime, listener, responses);
    }
}

Example 4 with DocWriteRequest

use of org.elasticsearch.action.DocWriteRequest in project elasticsearch by elastic.

the class TransportShardBulkAction method updateReplicaRequest.

// Visible for unit testing
static Translog.Location updateReplicaRequest(BulkItemResultHolder bulkItemResult, final DocWriteRequest.OpType opType, final Translog.Location originalLocation, BulkShardRequest request) {
    final Engine.Result operationResult = bulkItemResult.operationResult;
    final DocWriteResponse response = bulkItemResult.response;
    final BulkItemRequest replicaRequest = bulkItemResult.replicaRequest;
    if (operationResult == null) {
        // in case of noop update operation
        assert response.getResult() == DocWriteResponse.Result.NOOP : "only noop updates can have a null operation";
        replicaRequest.setPrimaryResponse(new BulkItemResponse(replicaRequest.id(), opType, response));
        return originalLocation;
    } else if (operationResult.hasFailure() == false) {
        BulkItemResponse primaryResponse = new BulkItemResponse(replicaRequest.id(), opType, response);
        replicaRequest.setPrimaryResponse(primaryResponse);
        // set a blank ShardInfo so we can safely send it to the replicas. We won't use it in the real response though.
        primaryResponse.getResponse().setShardInfo(new ShardInfo());
        // The operation was successful, advance the translog
        return locationToSync(originalLocation, operationResult.getTranslogLocation());
    } else {
        DocWriteRequest docWriteRequest = replicaRequest.request();
        Exception failure = operationResult.getFailure();
        if (isConflictException(failure)) {
            logger.trace((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}", request.shardId(), docWriteRequest.opType().getLowercase(), request), failure);
        } else {
            logger.debug((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}", request.shardId(), docWriteRequest.opType().getLowercase(), request), failure);
        }
        // then just use the response we got from the failed execution
        if (replicaRequest.getPrimaryResponse() == null || isConflictException(failure) == false) {
            replicaRequest.setPrimaryResponse(new BulkItemResponse(replicaRequest.id(), docWriteRequest.opType(), // concrete index instead of an alias if used!
            new BulkItemResponse.Failure(request.index(), docWriteRequest.type(), docWriteRequest.id(), failure)));
        }
        return originalLocation;
    }
}

Example 5 with DocWriteRequest

use of org.elasticsearch.action.DocWriteRequest in project elasticsearch by elastic.

the class TransportSingleItemBulkWriteAction method shardOperationOnPrimary.

@Override
protected WritePrimaryResult<Request, Response> shardOperationOnPrimary(Request request, final IndexShard primary) throws Exception {
    BulkItemRequest[] itemRequests = new BulkItemRequest[1];
    WriteRequest.RefreshPolicy refreshPolicy = request.getRefreshPolicy();
    request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
    itemRequests[0] = new BulkItemRequest(0, ((DocWriteRequest) request));
    BulkShardRequest bulkShardRequest = new BulkShardRequest(request.shardId(), refreshPolicy, itemRequests);
    WritePrimaryResult<BulkShardRequest, BulkShardResponse> bulkResult = shardBulkAction.shardOperationOnPrimary(bulkShardRequest, primary);
    assert bulkResult.finalResponseIfSuccessful.getResponses().length == 1 : "expected only one bulk shard response";
    BulkItemResponse itemResponse = bulkResult.finalResponseIfSuccessful.getResponses()[0];
    final Response response;
    final Exception failure;
    if (itemResponse.isFailed()) {
        failure = itemResponse.getFailure().getCause();
        response = null;
    } else {
        response = (Response) itemResponse.getResponse();
        failure = null;
    }
    return new WritePrimaryResult<>(request, response, bulkResult.location, failure, primary, logger);
}