Examples with BulkRequest org.elasticsearch.action.bulk.BulkRequest used on opensource projects

Example 11 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class ParentTaskAssigningClientTests method testSetsParentId.

public void testSetsParentId() {
    TaskId[] parentTaskId = new TaskId[] { new TaskId(randomAsciiOfLength(3), randomLong()) };
    // This mock will do nothing but verify that parentTaskId is set on all requests sent to it.
    NoOpClient mock = new NoOpClient(getTestName()) {

        @Override
        protected <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
            assertEquals(parentTaskId[0], request.getParentTask());
            super.doExecute(action, request, listener);
        }
    };
    try (ParentTaskAssigningClient client = new ParentTaskAssigningClient(mock, parentTaskId[0])) {
        // All of these should have the parentTaskId set
        client.bulk(new BulkRequest());
        client.search(new SearchRequest());
        client.clearScroll(new ClearScrollRequest());
        // Now lets verify that unwrapped calls don't have the parentTaskId set
        parentTaskId[0] = TaskId.EMPTY_TASK_ID;
        client.unwrap().bulk(new BulkRequest());
        client.unwrap().search(new SearchRequest());
        client.unwrap().clearScroll(new ClearScrollRequest());
    }
}

Example 12 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project samza by apache.

the class ElasticsearchSystemProducer method register.

@Override
public void register(final String source) {
    BulkProcessor.Listener listener = new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
        // Nothing to do.
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            boolean hasFatalError = false;
            //Do not consider version conficts to be errors. Ignore old versions
            if (response.hasFailures()) {
                for (BulkItemResponse itemResp : response.getItems()) {
                    if (itemResp.isFailed()) {
                        if (itemResp.getFailure().getStatus().equals(RestStatus.CONFLICT)) {
                            LOGGER.info("Failed to index document in Elasticsearch: " + itemResp.getFailureMessage());
                        } else {
                            hasFatalError = true;
                            LOGGER.error("Failed to index document in Elasticsearch: " + itemResp.getFailureMessage());
                        }
                    }
                }
            }
            if (hasFatalError) {
                sendFailed.set(true);
            } else {
                updateSuccessMetrics(response);
            }
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            LOGGER.error(failure.getMessage());
            thrown.compareAndSet(null, failure);
            sendFailed.set(true);
        }

        private void updateSuccessMetrics(BulkResponse response) {
            metrics.bulkSendSuccess.inc();
            int writes = 0;
            for (BulkItemResponse itemResp : response.getItems()) {
                if (itemResp.isFailed()) {
                    if (itemResp.getFailure().getStatus().equals(RestStatus.CONFLICT)) {
                        metrics.conflicts.inc();
                    }
                } else {
                    ActionResponse resp = itemResp.getResponse();
                    if (resp instanceof IndexResponse) {
                        writes += 1;
                        if (((IndexResponse) resp).isCreated()) {
                            metrics.inserts.inc();
                        } else {
                            metrics.updates.inc();
                        }
                    } else {
                        LOGGER.error("Unexpected Elasticsearch action response type: " + resp.getClass().getSimpleName());
                    }
                }
            }
            LOGGER.info(String.format("Wrote %s messages from %s to %s.", writes, source, system));
        }
    };
    sourceBulkProcessor.put(source, bulkProcessorFactory.getBulkProcessor(client, listener));
}

Example 13 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class RestBulkAction method prepareRequest.

@Override
public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
    BulkRequest bulkRequest = Requests.bulkRequest();
    String defaultIndex = request.param("index");
    String defaultType = request.param("type");
    String defaultRouting = request.param("routing");
    FetchSourceContext defaultFetchSourceContext = FetchSourceContext.parseFromRestRequest(request);
    String fieldsParam = request.param("fields");
    if (fieldsParam != null) {
        DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead");
    }
    String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null;
    String defaultPipeline = request.param("pipeline");
    String waitForActiveShards = request.param("wait_for_active_shards");
    if (waitForActiveShards != null) {
        bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
    }
    bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
    bulkRequest.setRefreshPolicy(request.param("refresh"));
    bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType());
    return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel));
}

Example 14 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class RestNoopBulkAction method prepareRequest.

@Override
public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
    BulkRequest bulkRequest = Requests.bulkRequest();
    String defaultIndex = request.param("index");
    String defaultType = request.param("type");
    String defaultRouting = request.param("routing");
    String fieldsParam = request.param("fields");
    String defaultPipeline = request.param("pipeline");
    String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null;
    String waitForActiveShards = request.param("wait_for_active_shards");
    if (waitForActiveShards != null) {
        bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
    }
    bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
    bulkRequest.setRefreshPolicy(request.param("refresh"));
    bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true, request.getXContentType());
    // short circuit the call to the transport layer
    return channel -> {
        BulkRestBuilderListener listener = new BulkRestBuilderListener(channel, request);
        listener.onResponse(bulkRequest);
    };
}

Example 15 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class AsyncBulkByScrollActionTests method bulkRetryTestCase.

/**
     * Execute a bulk retry test case. The total number of failures is random and the number of retries attempted is set to
     * testRequest.getMaxRetries and controled by the failWithRejection parameter.
     */
private void bulkRetryTestCase(boolean failWithRejection) throws Exception {
    int totalFailures = randomIntBetween(1, testRequest.getMaxRetries());
    int size = randomIntBetween(1, 100);
    testRequest.setMaxRetries(totalFailures - (failWithRejection ? 1 : 0));
    client.bulksToReject = client.bulksAttempts.get() + totalFailures;
    /*
         * When we get a successful bulk response we usually start the next scroll request but lets just intercept that so we don't have to
         * deal with it. We just wait for it to happen.
         */
    CountDownLatch successLatch = new CountDownLatch(1);
    DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff() {

        @Override
        void startNextScroll(TimeValue lastBatchStartTime, int lastBatchSize) {
            successLatch.countDown();
        }
    };
    BulkRequest request = new BulkRequest();
    for (int i = 0; i < size + 1; i++) {
        request.add(new IndexRequest("index", "type", "id" + i));
    }
    action.sendBulkRequest(timeValueNanos(System.nanoTime()), request);
    if (failWithRejection) {
        BulkByScrollResponse response = listener.get();
        assertThat(response.getBulkFailures(), hasSize(1));
        assertEquals(response.getBulkFailures().get(0).getStatus(), RestStatus.TOO_MANY_REQUESTS);
        assertThat(response.getSearchFailures(), empty());
        assertNull(response.getReasonCancelled());
    } else {
        assertTrue(successLatch.await(10, TimeUnit.SECONDS));
    }
}