Examples with BulkRequestBuilder org.elasticsearch.action.bulk.BulkRequestBuilder used on opensource projects

Example 46 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project fess by codelibs.

the class EsAbstractBehavior method delegateBatchRequest.

protected <BUILDER> int[] delegateBatchRequest(final List<? extends Entity> entityList, Function<EsAbstractEntity, BUILDER> call) {
    @SuppressWarnings("unchecked") final BulkList<? extends Entity, BUILDER> bulkList = (BulkList<? extends Entity, BUILDER>) entityList;
    final RequestOptionCall<BUILDER> builderEntityCall = bulkList.getEntityCall();
    final BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (final Entity entity : entityList) {
        final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
        BUILDER builder = call.apply(esEntity);
        if (builder instanceof IndexRequestBuilder) {
            if (builderEntityCall != null) {
                builderEntityCall.callback(builder);
            }
            bulkBuilder.add((IndexRequestBuilder) builder);
        } else if (builder instanceof UpdateRequestBuilder) {
            if (builderEntityCall != null) {
                builderEntityCall.callback(builder);
            }
            bulkBuilder.add((UpdateRequestBuilder) builder);
        } else if (builder instanceof DeleteRequestBuilder) {
            if (builderEntityCall != null) {
                builderEntityCall.callback(builder);
            }
            bulkBuilder.add((DeleteRequestBuilder) builder);
        }
    }
    final RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
    if (builderCall != null) {
        builderCall.callback(bulkBuilder);
    }
    final BulkResponse response = bulkBuilder.execute().actionGet(bulkTimeout);
    final BulkItemResponse[] itemResponses = response.getItems();
    if (itemResponses.length != entityList.size()) {
        throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
    }
    final int[] results = new int[itemResponses.length];
    for (int i = 0; i < itemResponses.length; i++) {
        final BulkItemResponse itemResponse = itemResponses[i];
        final Entity entity = entityList.get(i);
        if (entity instanceof EsAbstractEntity) {
            ((EsAbstractEntity) entity).asDocMeta().id(itemResponse.getId());
        }
        results[i] = itemResponse.isFailed() ? 0 : 1;
    }
    return results;
}

Example 47 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project apex-malhar by apache.

the class AbstractElasticSearchOutputOperator method processBatch.

/**
 * This will flush all the tuples from queue to ElasticSearch.
 * It uses BulkRequestBuilder API for sending batch.
 */
private void processBatch() {
    BulkRequestBuilder bulkRequestBuilder = new BulkRequestBuilder(store.client);
    while (!tupleBatch.isEmpty()) {
        T tuple = tupleBatch.remove();
        IndexRequestBuilder indexRequestBuilder = getIndexRequestBuilder(tuple);
        bulkRequestBuilder.add(indexRequestBuilder);
    }
    BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
    if (bulkResponse.hasFailures()) {
        DTThrowable.rethrow(new Exception(bulkResponse.buildFailureMessage()));
    }
}

Example 48 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project metron by apache.

the class ElasticsearchDao method batchUpdate.

@Override
public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
    String indexPostfix = ElasticsearchUtils.getIndexFormat(accessConfig.getGlobalConfigSupplier().get()).format(new Date());
    BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    // Get the indices we'll actually be using for each Document.
    for (Map.Entry<Document, Optional<String>> updateEntry : updates.entrySet()) {
        Document update = updateEntry.getKey();
        String sensorType = update.getSensorType();
        String indexName = getIndexName(update, updateEntry.getValue(), indexPostfix);
        IndexRequest indexRequest = buildIndexRequest(update, sensorType, indexName);
        bulkRequestBuilder.add(indexRequest);
    }
    BulkResponse bulkResponse = bulkRequestBuilder.get();
    if (bulkResponse.hasFailures()) {
        LOG.error("Bulk Request has failures: {}", bulkResponse.buildFailureMessage());
        throw new IOException("ElasticsearchDao upsert failed: " + bulkResponse.buildFailureMessage());
    }
}

Example 49 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project metron by apache.

the class ElasticsearchWriter method write.

@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
    final String indexPostfix = dateFormat.format(new Date());
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (JSONObject message : messages) {
        JSONObject esDoc = new JSONObject();
        for (Object k : message.keySet()) {
            deDot(k.toString(), message, esDoc);
        }
        String indexName = ElasticsearchUtils.getIndexName(sensorType, indexPostfix, configurations);
        IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName, sensorType + "_doc");
        indexRequestBuilder = indexRequestBuilder.setSource(esDoc.toJSONString());
        String guid = (String) esDoc.get(Constants.GUID);
        if (guid != null) {
            indexRequestBuilder.setId(guid);
        }
        Object ts = esDoc.get("timestamp");
        if (ts != null) {
            indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString());
        }
        bulkRequest.add(indexRequestBuilder);
    }
    BulkResponse bulkResponse = bulkRequest.execute().actionGet();
    return buildWriteReponse(tuples, bulkResponse);
}

Example 50 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project metron by apache.

the class ElasticSearchComponent method add.

public BulkResponse add(String indexName, String sensorType, Iterable<String> docs) throws IOException {
    BulkRequestBuilder bulkRequest = getClient().prepareBulk();
    for (String doc : docs) {
        IndexRequestBuilder indexRequestBuilder = getClient().prepareIndex(indexName, sensorType + "_doc");
        indexRequestBuilder = indexRequestBuilder.setSource(doc);
        Map<String, Object> esDoc = JSONUtils.INSTANCE.load(doc, JSONUtils.MAP_SUPPLIER);
        indexRequestBuilder.setId((String) esDoc.get(Constants.GUID));
        Object ts = esDoc.get("timestamp");
        if (ts != null) {
            indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString());
        }
        bulkRequest.add(indexRequestBuilder);
    }
    BulkResponse response = bulkRequest.execute().actionGet();
    if (response.hasFailures()) {
        throw new IOException(response.buildFailureMessage());
    }
    return response;
}