Examples with BulkRequest org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest used on opensource projects

Example 1 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class Request method bulk.

static Request bulk(BulkRequest bulkRequest) throws IOException {
    Params parameters = Params.builder();
    parameters.withTimeout(bulkRequest.timeout());
    parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
    // Bulk API only supports newline delimited JSON or Smile. Before executing
    // the bulk, we need to check that all requests have the same content-type
    // and this content-type is supported by the Bulk API.
    XContentType bulkContentType = null;
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> request = bulkRequest.requests().get(i);
        DocWriteRequest.OpType opType = request.opType();
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            bulkContentType = enforceSameContentType((IndexRequest) request, bulkContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) request;
            if (updateRequest.doc() != null) {
                bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
            }
            if (updateRequest.upsertRequest() != null) {
                bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
            }
        }
    }
    if (bulkContentType == null) {
        bulkContentType = XContentType.JSON;
    }
    byte separator = bulkContentType.xContent().streamSeparator();
    ContentType requestContentType = ContentType.create(bulkContentType.mediaType());
    ByteArrayOutputStream content = new ByteArrayOutputStream();
    for (DocWriteRequest<?> request : bulkRequest.requests()) {
        DocWriteRequest.OpType opType = request.opType();
        try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
            metadata.startObject();
            {
                metadata.startObject(opType.getLowercase());
                if (Strings.hasLength(request.index())) {
                    metadata.field("_index", request.index());
                }
                if (Strings.hasLength(request.type())) {
                    metadata.field("_type", request.type());
                }
                if (Strings.hasLength(request.id())) {
                    metadata.field("_id", request.id());
                }
                if (Strings.hasLength(request.routing())) {
                    metadata.field("_routing", request.routing());
                }
                if (Strings.hasLength(request.parent())) {
                    metadata.field("_parent", request.parent());
                }
                if (request.version() != Versions.MATCH_ANY) {
                    metadata.field("_version", request.version());
                }
                VersionType versionType = request.versionType();
                if (versionType != VersionType.INTERNAL) {
                    if (versionType == VersionType.EXTERNAL) {
                        metadata.field("_version_type", "external");
                    } else if (versionType == VersionType.EXTERNAL_GTE) {
                        metadata.field("_version_type", "external_gte");
                    } else if (versionType == VersionType.FORCE) {
                        metadata.field("_version_type", "force");
                    }
                }
                if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
                    IndexRequest indexRequest = (IndexRequest) request;
                    if (Strings.hasLength(indexRequest.getPipeline())) {
                        metadata.field("pipeline", indexRequest.getPipeline());
                    }
                } else if (opType == DocWriteRequest.OpType.UPDATE) {
                    UpdateRequest updateRequest = (UpdateRequest) request;
                    if (updateRequest.retryOnConflict() > 0) {
                        metadata.field("_retry_on_conflict", updateRequest.retryOnConflict());
                    }
                    if (updateRequest.fetchSource() != null) {
                        metadata.field("_source", updateRequest.fetchSource());
                    }
                }
                metadata.endObject();
            }
            metadata.endObject();
            BytesRef metadataSource = metadata.bytes().toBytesRef();
            content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
            content.write(separator);
        }
        BytesRef source = null;
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest indexRequest = (IndexRequest) request;
            BytesReference indexSource = indexRequest.source();
            XContentType indexXContentType = indexRequest.getContentType();
            try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, indexSource, indexXContentType)) {
                try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                    builder.copyCurrentStructure(parser);
                    source = builder.bytes().toBytesRef();
                }
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            source = XContentHelper.toXContent((UpdateRequest) request, bulkContentType, false).toBytesRef();
        }
        if (source != null) {
            content.write(source.bytes, source.offset, source.length);
            content.write(separator);
        }
    }
    HttpEntity entity = new ByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType);
    return new Request(HttpPost.METHOD_NAME, "/_bulk", parameters.getParams(), entity);
}

Example 2 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class AbstractAsyncBulkByScrollAction method prepareBulkRequest.

/**
     * Prepare the bulk request. Called on the generic thread pool after some preflight checks have been done one the SearchResponse and any
     * delay has been slept. Uses the generic thread pool because reindex is rare enough not to need its own thread pool and because the
     * thread may be blocked by the user script.
     */
void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.Response response) {
    if (task.isCancelled()) {
        finishHim(null);
        return;
    }
    if (response.getHits().isEmpty()) {
        refreshAndFinish(emptyList(), emptyList(), false);
        return;
    }
    task.countBatch();
    List<? extends ScrollableHitSource.Hit> hits = response.getHits();
    if (mainRequest.getSize() != SIZE_ALL_MATCHES) {
        // Truncate the hits if we have more than the request size
        long remaining = max(0, mainRequest.getSize() - task.getSuccessfullyProcessed());
        if (remaining < hits.size()) {
            hits = hits.subList(0, (int) remaining);
        }
    }
    BulkRequest request = buildBulk(hits);
    if (request.requests().isEmpty()) {
        /*
             * If we noop-ed the entire batch then just skip to the next batch or the BulkRequest would fail validation.
             */
        startNextScroll(thisBatchStartTime, 0);
        return;
    }
    request.timeout(mainRequest.getTimeout());
    request.waitForActiveShards(mainRequest.getWaitForActiveShards());
    if (logger.isDebugEnabled()) {
        logger.debug("sending [{}] entry, [{}] bulk request", request.requests().size(), new ByteSizeValue(request.estimatedSizeInBytes()));
    }
    sendBulkRequest(thisBatchStartTime, request);
}

Example 3 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class ParentTaskAssigningClientTests method testSetsParentId.

public void testSetsParentId() {
    TaskId[] parentTaskId = new TaskId[] { new TaskId(randomAsciiOfLength(3), randomLong()) };
    // This mock will do nothing but verify that parentTaskId is set on all requests sent to it.
    NoOpClient mock = new NoOpClient(getTestName()) {

        @Override
        protected <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
            assertEquals(parentTaskId[0], request.getParentTask());
            super.doExecute(action, request, listener);
        }
    };
    try (ParentTaskAssigningClient client = new ParentTaskAssigningClient(mock, parentTaskId[0])) {
        // All of these should have the parentTaskId set
        client.bulk(new BulkRequest());
        client.search(new SearchRequest());
        client.clearScroll(new ClearScrollRequest());
        // Now lets verify that unwrapped calls don't have the parentTaskId set
        parentTaskId[0] = TaskId.EMPTY_TASK_ID;
        client.unwrap().bulk(new BulkRequest());
        client.unwrap().search(new SearchRequest());
        client.unwrap().clearScroll(new ClearScrollRequest());
    }
}

Example 4 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class IngestClientIT method testBulkWithIngestFailures.

public void testBulkWithIngestFailures() throws Exception {
    createIndex("index");
    BytesReference source = jsonBuilder().startObject().field("description", "my_pipeline").startArray("processors").startObject().startObject("test").endObject().endObject().endArray().endObject().bytes();
    PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
    client().admin().cluster().putPipeline(putPipelineRequest).get();
    int numRequests = scaledRandomIntBetween(32, 128);
    BulkRequest bulkRequest = new BulkRequest();
    for (int i = 0; i < numRequests; i++) {
        IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i)).setPipeline("_id");
        indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", i % 2 == 0);
        bulkRequest.add(indexRequest);
    }
    BulkResponse response = client().bulk(bulkRequest).actionGet();
    assertThat(response.getItems().length, equalTo(bulkRequest.requests().size()));
    for (int i = 0; i < bulkRequest.requests().size(); i++) {
        BulkItemResponse itemResponse = response.getItems()[i];
        if (i % 2 == 0) {
            BulkItemResponse.Failure failure = itemResponse.getFailure();
            ElasticsearchException compoundProcessorException = (ElasticsearchException) failure.getCause();
            assertThat(compoundProcessorException.getRootCause().getMessage(), equalTo("test processor failed"));
        } else {
            IndexResponse indexResponse = itemResponse.getResponse();
            assertThat("Expected a successful response but found failure [" + itemResponse.getFailure() + "].", itemResponse.isFailed(), is(false));
            assertThat(indexResponse, notNullValue());
            assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
            assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
        }
    }
}

Example 5 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class PipelineExecutionServiceTests method testBulkRequestExecution.

public void testBulkRequestExecution() throws Exception {
    BulkRequest bulkRequest = new BulkRequest();
    String pipelineId = "_id";
    int numRequest = scaledRandomIntBetween(8, 64);
    for (int i = 0; i < numRequest; i++) {
        IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId);
        indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
        bulkRequest.add(indexRequest);
    }
    when(store.get(pipelineId)).thenReturn(new Pipeline(pipelineId, null, version, new CompoundProcessor()));
    @SuppressWarnings("unchecked") BiConsumer<IndexRequest, Exception> requestItemErrorHandler = mock(BiConsumer.class);
    @SuppressWarnings("unchecked") Consumer<Exception> completionHandler = mock(Consumer.class);
    executionService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler);
    verify(requestItemErrorHandler, never()).accept(any(), any());
    verify(completionHandler, times(1)).accept(null);
}