Examples with BulkRequest org.elasticsearch.action.bulk.BulkRequest used on opensource projects

Example 16 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class RestBulkAction method prepareRequest.

@Override
public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
    BulkRequest bulkRequest = Requests.bulkRequest();
    String defaultIndex = request.param("index");
    String defaultType = request.param("type");
    String defaultRouting = request.param("routing");
    FetchSourceContext defaultFetchSourceContext = FetchSourceContext.parseFromRestRequest(request);
    String fieldsParam = request.param("fields");
    if (fieldsParam != null) {
        DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead");
    }
    String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null;
    String defaultPipeline = request.param("pipeline");
    String waitForActiveShards = request.param("wait_for_active_shards");
    if (waitForActiveShards != null) {
        bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
    }
    bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
    bulkRequest.setRefreshPolicy(request.param("refresh"));
    bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType());
    return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel));
}

Example 17 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class CircuitBreakerServiceIT method testLimitsRequestSize.

public void testLimitsRequestSize() throws Exception {
    ByteSizeValue inFlightRequestsLimit = new ByteSizeValue(8, ByteSizeUnit.KB);
    if (noopBreakerUsed()) {
        logger.info("--> noop breakers used, skipping test");
        return;
    }
    internalCluster().ensureAtLeastNumDataNodes(2);
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    List<NodeStats> dataNodeStats = new ArrayList<>();
    for (NodeStats stat : nodeStats.getNodes()) {
        if (stat.getNode().isDataNode()) {
            dataNodeStats.add(stat);
        }
    }
    assertThat(dataNodeStats.size(), greaterThanOrEqualTo(2));
    Collections.shuffle(dataNodeStats, random());
    // send bulk request from source node to target node later. The sole shard is bound to the target node.
    NodeStats targetNode = dataNodeStats.get(0);
    NodeStats sourceNode = dataNodeStats.get(1);
    assertAcked(prepareCreate("index").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put("index.routing.allocation.include._name", targetNode.getNode().getName()).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)));
    Client client = client(sourceNode.getNode().getName());
    // we use the limit size as a (very) rough indication on how many requests we should sent to hit the limit
    int numRequests = inFlightRequestsLimit.bytesAsInt();
    BulkRequest bulkRequest = new BulkRequest();
    for (int i = 0; i < numRequests; i++) {
        IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i));
        indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "num", i);
        bulkRequest.add(indexRequest);
    }
    Settings limitSettings = Settings.builder().put(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), inFlightRequestsLimit).build();
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(limitSettings));
    // can either fail directly with an exception or the response contains exceptions (depending on client)
    try {
        BulkResponse response = client.bulk(bulkRequest).actionGet();
        if (!response.hasFailures()) {
            fail("Should have thrown CircuitBreakingException");
        } else {
            // each item must have failed with CircuitBreakingException
            for (BulkItemResponse bulkItemResponse : response) {
                Throwable cause = ExceptionsHelper.unwrapCause(bulkItemResponse.getFailure().getCause());
                assertThat(cause, instanceOf(CircuitBreakingException.class));
                assertEquals(((CircuitBreakingException) cause).getByteLimit(), inFlightRequestsLimit.getBytes());
            }
        }
    } catch (CircuitBreakingException ex) {
        assertEquals(ex.getByteLimit(), inFlightRequestsLimit.getBytes());
    }
}

Example 18 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch-river-couchdb by elastic.

the class CouchdbRiver method start.

@Override
public void start() {
    logger.info("starting couchdb stream: host [{}], port [{}], filter [{}], db [{}], indexing to [{}]/[{}]", couchHost, couchPort, couchFilter, couchDb, indexName, typeName);
    // Creating bulk processor
    this.bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            logger.debug("Going to execute new bulk composed of {} actions", request.numberOfActions());
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            logger.debug("Executed bulk composed of {} actions", request.numberOfActions());
            if (response.hasFailures()) {
                logger.warn("There was failures while executing bulk", response.buildFailureMessage());
                if (logger.isDebugEnabled()) {
                    for (BulkItemResponse item : response.getItems()) {
                        if (item.isFailed()) {
                            logger.debug("Error for {}/{}/{} for {} operation: {}", item.getIndex(), item.getType(), item.getId(), item.getOpType(), item.getFailureMessage());
                        }
                    }
                }
            }
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            logger.warn("Error executing bulk", failure);
        }
    }).setBulkActions(bulkSize).setConcurrentRequests(maxConcurrentBulk).setFlushInterval(bulkFlushInterval).build();
    slurperThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "couchdb_river_slurper").newThread(new Slurper());
    indexerThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "couchdb_river_indexer").newThread(new Indexer());
    indexerThread.start();
    slurperThread.start();
}

Example 19 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class RestNoopBulkAction method prepareRequest.

@Override
public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
    BulkRequest bulkRequest = Requests.bulkRequest();
    String defaultIndex = request.param("index");
    String defaultType = request.param("type");
    String defaultRouting = request.param("routing");
    String fieldsParam = request.param("fields");
    String defaultPipeline = request.param("pipeline");
    String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null;
    String waitForActiveShards = request.param("wait_for_active_shards");
    if (waitForActiveShards != null) {
        bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
    }
    bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
    bulkRequest.setRefreshPolicy(request.param("refresh"));
    bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true, request.getXContentType());
    // short circuit the call to the transport layer
    return channel -> {
        BulkRestBuilderListener listener = new BulkRestBuilderListener(channel, request);
        listener.onResponse(bulkRequest);
    };
}

Example 20 with BulkRequest

use of org.elasticsearch.action.bulk.BulkRequest in project elasticsearch by elastic.

the class IndicesRequestIT method testBulk.

public void testBulk() {
    String[] bulkShardActions = new String[] { BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]" };
    interceptTransportActions(bulkShardActions);
    List<String> indices = new ArrayList<>();
    BulkRequest bulkRequest = new BulkRequest();
    int numIndexRequests = iterations(1, 10);
    for (int i = 0; i < numIndexRequests; i++) {
        String indexOrAlias = randomIndexOrAlias();
        bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
        indices.add(indexOrAlias);
    }
    int numDeleteRequests = iterations(1, 10);
    for (int i = 0; i < numDeleteRequests; i++) {
        String indexOrAlias = randomIndexOrAlias();
        bulkRequest.add(new DeleteRequest(indexOrAlias, "type", "id"));
        indices.add(indexOrAlias);
    }
    int numUpdateRequests = iterations(1, 10);
    for (int i = 0; i < numUpdateRequests; i++) {
        String indexOrAlias = randomIndexOrAlias();
        bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"));
        indices.add(indexOrAlias);
    }
    internalCluster().coordOnlyNodeClient().bulk(bulkRequest).actionGet();
    clearInterceptedActions();
    assertIndicesSubset(indices, bulkShardActions);
}