Examples with BulkRequest org.elasticsearch.action.bulk.BulkRequest used on opensource projects

Example 6 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class PipelineExecutionServiceTests method testBulkRequestExecutionWithFailures.

public void testBulkRequestExecutionWithFailures() throws Exception {
    BulkRequest bulkRequest = new BulkRequest();
    String pipelineId = "_id";
    int numRequest = scaledRandomIntBetween(8, 64);
    int numIndexRequests = 0;
    for (int i = 0; i < numRequest; i++) {
        DocWriteRequest request;
        if (randomBoolean()) {
            if (randomBoolean()) {
                request = new DeleteRequest("_index", "_type", "_id");
            } else {
                request = new UpdateRequest("_index", "_type", "_id");
            }
        } else {
            IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId);
            indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
            request = indexRequest;
            numIndexRequests++;
        }
        bulkRequest.add(request);
    }
    CompoundProcessor processor = mock(CompoundProcessor.class);
    when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class)));
    Exception error = new RuntimeException();
    doThrow(error).when(processor).execute(any());
    when(store.get(pipelineId)).thenReturn(new Pipeline(pipelineId, null, version, processor));
    @SuppressWarnings("unchecked") BiConsumer<IndexRequest, Exception> requestItemErrorHandler = mock(BiConsumer.class);
    @SuppressWarnings("unchecked") Consumer<Exception> completionHandler = mock(Consumer.class);
    executionService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler);
    verify(requestItemErrorHandler, times(numIndexRequests)).accept(any(IndexRequest.class), eq(error));
    verify(completionHandler, times(1)).accept(null);
}

Example 7 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class CrudIT method testBulk.

public void testBulk() throws IOException {
    int nbItems = randomIntBetween(10, 100);
    boolean[] errors = new boolean[nbItems];
    XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
    BulkRequest bulkRequest = new BulkRequest();
    for (int i = 0; i < nbItems; i++) {
        String id = String.valueOf(i);
        boolean erroneous = randomBoolean();
        errors[i] = erroneous;
        DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
        if (opType == DocWriteRequest.OpType.DELETE) {
            if (erroneous == false) {
                assertEquals(RestStatus.CREATED, highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status());
            }
            DeleteRequest deleteRequest = new DeleteRequest("index", "test", id);
            bulkRequest.add(deleteRequest);
        } else {
            BytesReference source = XContentBuilder.builder(xContentType.xContent()).startObject().field("id", i).endObject().bytes();
            if (opType == DocWriteRequest.OpType.INDEX) {
                IndexRequest indexRequest = new IndexRequest("index", "test", id).source(source, xContentType);
                if (erroneous) {
                    indexRequest.version(12L);
                }
                bulkRequest.add(indexRequest);
            } else if (opType == DocWriteRequest.OpType.CREATE) {
                IndexRequest createRequest = new IndexRequest("index", "test", id).source(source, xContentType).create(true);
                if (erroneous) {
                    assertEquals(RestStatus.CREATED, highLevelClient().index(createRequest).status());
                }
                bulkRequest.add(createRequest);
            } else if (opType == DocWriteRequest.OpType.UPDATE) {
                UpdateRequest updateRequest = new UpdateRequest("index", "test", id).doc(new IndexRequest().source(source, xContentType));
                if (erroneous == false) {
                    assertEquals(RestStatus.CREATED, highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status());
                }
                bulkRequest.add(updateRequest);
            }
        }
    }
    BulkResponse bulkResponse = execute(bulkRequest, highLevelClient()::bulk, highLevelClient()::bulkAsync);
    assertEquals(RestStatus.OK, bulkResponse.status());
    assertTrue(bulkResponse.getTookInMillis() > 0);
    assertEquals(nbItems, bulkResponse.getItems().length);
    for (int i = 0; i < nbItems; i++) {
        BulkItemResponse bulkItemResponse = bulkResponse.getItems()[i];
        assertEquals(i, bulkItemResponse.getItemId());
        assertEquals("index", bulkItemResponse.getIndex());
        assertEquals("test", bulkItemResponse.getType());
        assertEquals(String.valueOf(i), bulkItemResponse.getId());
        DocWriteRequest.OpType requestOpType = bulkRequest.requests().get(i).opType();
        if (requestOpType == DocWriteRequest.OpType.INDEX || requestOpType == DocWriteRequest.OpType.CREATE) {
            assertEquals(errors[i], bulkItemResponse.isFailed());
            assertEquals(errors[i] ? RestStatus.CONFLICT : RestStatus.CREATED, bulkItemResponse.status());
        } else if (requestOpType == DocWriteRequest.OpType.UPDATE) {
            assertEquals(errors[i], bulkItemResponse.isFailed());
            assertEquals(errors[i] ? RestStatus.NOT_FOUND : RestStatus.OK, bulkItemResponse.status());
        } else if (requestOpType == DocWriteRequest.OpType.DELETE) {
            assertFalse(bulkItemResponse.isFailed());
            assertEquals(errors[i] ? RestStatus.NOT_FOUND : RestStatus.OK, bulkItemResponse.status());
        }
    }
}

Example 8 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class RequestTests method testBulk.

public void testBulk() throws IOException {
    Map<String, String> expectedParams = new HashMap<>();
    BulkRequest bulkRequest = new BulkRequest();
    if (randomBoolean()) {
        String timeout = randomTimeValue();
        bulkRequest.timeout(timeout);
        expectedParams.put("timeout", timeout);
    } else {
        expectedParams.put("timeout", BulkShardRequest.DEFAULT_TIMEOUT.getStringRep());
    }
    if (randomBoolean()) {
        WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
        bulkRequest.setRefreshPolicy(refreshPolicy);
        if (refreshPolicy != WriteRequest.RefreshPolicy.NONE) {
            expectedParams.put("refresh", refreshPolicy.getValue());
        }
    }
    XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
    int nbItems = randomIntBetween(10, 100);
    for (int i = 0; i < nbItems; i++) {
        String index = randomAsciiOfLength(5);
        String type = randomAsciiOfLength(5);
        String id = randomAsciiOfLength(5);
        BytesReference source = RandomObjects.randomSource(random(), xContentType);
        DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
        DocWriteRequest<?> docWriteRequest = null;
        if (opType == DocWriteRequest.OpType.INDEX) {
            IndexRequest indexRequest = new IndexRequest(index, type, id).source(source, xContentType);
            docWriteRequest = indexRequest;
            if (randomBoolean()) {
                indexRequest.setPipeline(randomAsciiOfLength(5));
            }
            if (randomBoolean()) {
                indexRequest.parent(randomAsciiOfLength(5));
            }
        } else if (opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest createRequest = new IndexRequest(index, type, id).source(source, xContentType).create(true);
            docWriteRequest = createRequest;
            if (randomBoolean()) {
                createRequest.parent(randomAsciiOfLength(5));
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            final UpdateRequest updateRequest = new UpdateRequest(index, type, id).doc(new IndexRequest().source(source, xContentType));
            docWriteRequest = updateRequest;
            if (randomBoolean()) {
                updateRequest.retryOnConflict(randomIntBetween(1, 5));
            }
            if (randomBoolean()) {
                randomizeFetchSourceContextParams(updateRequest::fetchSource, new HashMap<>());
            }
            if (randomBoolean()) {
                updateRequest.parent(randomAsciiOfLength(5));
            }
        } else if (opType == DocWriteRequest.OpType.DELETE) {
            docWriteRequest = new DeleteRequest(index, type, id);
        }
        if (randomBoolean()) {
            docWriteRequest.routing(randomAsciiOfLength(10));
        }
        if (randomBoolean()) {
            docWriteRequest.version(randomNonNegativeLong());
        }
        if (randomBoolean()) {
            docWriteRequest.versionType(randomFrom(VersionType.values()));
        }
        bulkRequest.add(docWriteRequest);
    }
    Request request = Request.bulk(bulkRequest);
    assertEquals("/_bulk", request.endpoint);
    assertEquals(expectedParams, request.params);
    assertEquals("POST", request.method);
    byte[] content = new byte[(int) request.entity.getContentLength()];
    try (InputStream inputStream = request.entity.getContent()) {
        Streams.readFully(inputStream, content);
    }
    BulkRequest parsedBulkRequest = new BulkRequest();
    parsedBulkRequest.add(content, 0, content.length, xContentType);
    assertEquals(bulkRequest.numberOfActions(), parsedBulkRequest.numberOfActions());
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> originalRequest = bulkRequest.requests().get(i);
        DocWriteRequest<?> parsedRequest = parsedBulkRequest.requests().get(i);
        assertEquals(originalRequest.opType(), parsedRequest.opType());
        assertEquals(originalRequest.index(), parsedRequest.index());
        assertEquals(originalRequest.type(), parsedRequest.type());
        assertEquals(originalRequest.id(), parsedRequest.id());
        assertEquals(originalRequest.routing(), parsedRequest.routing());
        assertEquals(originalRequest.parent(), parsedRequest.parent());
        assertEquals(originalRequest.version(), parsedRequest.version());
        assertEquals(originalRequest.versionType(), parsedRequest.versionType());
        DocWriteRequest.OpType opType = originalRequest.opType();
        if (opType == DocWriteRequest.OpType.INDEX) {
            IndexRequest indexRequest = (IndexRequest) originalRequest;
            IndexRequest parsedIndexRequest = (IndexRequest) parsedRequest;
            assertEquals(indexRequest.getPipeline(), parsedIndexRequest.getPipeline());
            assertToXContentEquivalent(indexRequest.source(), parsedIndexRequest.source(), xContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) originalRequest;
            UpdateRequest parsedUpdateRequest = (UpdateRequest) parsedRequest;
            assertEquals(updateRequest.retryOnConflict(), parsedUpdateRequest.retryOnConflict());
            assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource());
            if (updateRequest.doc() != null) {
                assertToXContentEquivalent(updateRequest.doc().source(), parsedUpdateRequest.doc().source(), xContentType);
            } else {
                assertNull(parsedUpdateRequest.doc());
            }
        }
    }
}

Example 9 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class Request method bulk.

static Request bulk(BulkRequest bulkRequest) throws IOException {
    Params parameters = Params.builder();
    parameters.withTimeout(bulkRequest.timeout());
    parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
    // Bulk API only supports newline delimited JSON or Smile. Before executing
    // the bulk, we need to check that all requests have the same content-type
    // and this content-type is supported by the Bulk API.
    XContentType bulkContentType = null;
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> request = bulkRequest.requests().get(i);
        DocWriteRequest.OpType opType = request.opType();
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            bulkContentType = enforceSameContentType((IndexRequest) request, bulkContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) request;
            if (updateRequest.doc() != null) {
                bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
            }
            if (updateRequest.upsertRequest() != null) {
                bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
            }
        }
    }
    if (bulkContentType == null) {
        bulkContentType = XContentType.JSON;
    }
    byte separator = bulkContentType.xContent().streamSeparator();
    ContentType requestContentType = ContentType.create(bulkContentType.mediaType());
    ByteArrayOutputStream content = new ByteArrayOutputStream();
    for (DocWriteRequest<?> request : bulkRequest.requests()) {
        DocWriteRequest.OpType opType = request.opType();
        try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
            metadata.startObject();
            {
                metadata.startObject(opType.getLowercase());
                if (Strings.hasLength(request.index())) {
                    metadata.field("_index", request.index());
                }
                if (Strings.hasLength(request.type())) {
                    metadata.field("_type", request.type());
                }
                if (Strings.hasLength(request.id())) {
                    metadata.field("_id", request.id());
                }
                if (Strings.hasLength(request.routing())) {
                    metadata.field("_routing", request.routing());
                }
                if (Strings.hasLength(request.parent())) {
                    metadata.field("_parent", request.parent());
                }
                if (request.version() != Versions.MATCH_ANY) {
                    metadata.field("_version", request.version());
                }
                VersionType versionType = request.versionType();
                if (versionType != VersionType.INTERNAL) {
                    if (versionType == VersionType.EXTERNAL) {
                        metadata.field("_version_type", "external");
                    } else if (versionType == VersionType.EXTERNAL_GTE) {
                        metadata.field("_version_type", "external_gte");
                    } else if (versionType == VersionType.FORCE) {
                        metadata.field("_version_type", "force");
                    }
                }
                if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
                    IndexRequest indexRequest = (IndexRequest) request;
                    if (Strings.hasLength(indexRequest.getPipeline())) {
                        metadata.field("pipeline", indexRequest.getPipeline());
                    }
                } else if (opType == DocWriteRequest.OpType.UPDATE) {
                    UpdateRequest updateRequest = (UpdateRequest) request;
                    if (updateRequest.retryOnConflict() > 0) {
                        metadata.field("_retry_on_conflict", updateRequest.retryOnConflict());
                    }
                    if (updateRequest.fetchSource() != null) {
                        metadata.field("_source", updateRequest.fetchSource());
                    }
                }
                metadata.endObject();
            }
            metadata.endObject();
            BytesRef metadataSource = metadata.bytes().toBytesRef();
            content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
            content.write(separator);
        }
        BytesRef source = null;
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest indexRequest = (IndexRequest) request;
            BytesReference indexSource = indexRequest.source();
            XContentType indexXContentType = indexRequest.getContentType();
            try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, indexSource, indexXContentType)) {
                try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                    builder.copyCurrentStructure(parser);
                    source = builder.bytes().toBytesRef();
                }
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            source = XContentHelper.toXContent((UpdateRequest) request, bulkContentType, false).toBytesRef();
        }
        if (source != null) {
            content.write(source.bytes, source.offset, source.length);
            content.write(separator);
        }
    }
    HttpEntity entity = new ByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType);
    return new Request(HttpPost.METHOD_NAME, "/_bulk", parameters.getParams(), entity);
}

Example 10 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class AbstractAsyncBulkByScrollAction method prepareBulkRequest.

/**
     * Prepare the bulk request. Called on the generic thread pool after some preflight checks have been done one the SearchResponse and any
     * delay has been slept. Uses the generic thread pool because reindex is rare enough not to need its own thread pool and because the
     * thread may be blocked by the user script.
     */
void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.Response response) {
    if (task.isCancelled()) {
        finishHim(null);
        return;
    }
    if (response.getHits().isEmpty()) {
        refreshAndFinish(emptyList(), emptyList(), false);
        return;
    }
    task.countBatch();
    List<? extends ScrollableHitSource.Hit> hits = response.getHits();
    if (mainRequest.getSize() != SIZE_ALL_MATCHES) {
        // Truncate the hits if we have more than the request size
        long remaining = max(0, mainRequest.getSize() - task.getSuccessfullyProcessed());
        if (remaining < hits.size()) {
            hits = hits.subList(0, (int) remaining);
        }
    }
    BulkRequest request = buildBulk(hits);
    if (request.requests().isEmpty()) {
        /*
             * If we noop-ed the entire batch then just skip to the next batch or the BulkRequest would fail validation.
             */
        startNextScroll(thisBatchStartTime, 0);
        return;
    }
    request.timeout(mainRequest.getTimeout());
    request.waitForActiveShards(mainRequest.getWaitForActiveShards());
    if (logger.isDebugEnabled()) {
        logger.debug("sending [{}] entry, [{}] bulk request", request.requests().size(), new ByteSizeValue(request.estimatedSizeInBytes()));
    }
    sendBulkRequest(thisBatchStartTime, request);
}