Examples with BulkResponse org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 71 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project metron by apache.

the class ElasticsearchDao method batchUpdate.

@Override
public void batchUpdate(Map<Document, Optional<String>> updates) throws IOException {
    String indexPostfix = ElasticsearchUtils.getIndexFormat(accessConfig.getGlobalConfigSupplier().get()).format(new Date());
    BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    // Get the indices we'll actually be using for each Document.
    for (Map.Entry<Document, Optional<String>> updateEntry : updates.entrySet()) {
        Document update = updateEntry.getKey();
        String sensorType = update.getSensorType();
        String indexName = getIndexName(update, updateEntry.getValue(), indexPostfix);
        IndexRequest indexRequest = buildIndexRequest(update, sensorType, indexName);
        bulkRequestBuilder.add(indexRequest);
    }
    BulkResponse bulkResponse = bulkRequestBuilder.get();
    if (bulkResponse.hasFailures()) {
        LOG.error("Bulk Request has failures: {}", bulkResponse.buildFailureMessage());
        throw new IOException("ElasticsearchDao upsert failed: " + bulkResponse.buildFailureMessage());
    }
}

Example 72 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project metron by apache.

the class ElasticsearchWriter method write.

@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception {
    final String indexPostfix = dateFormat.format(new Date());
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (JSONObject message : messages) {
        JSONObject esDoc = new JSONObject();
        for (Object k : message.keySet()) {
            deDot(k.toString(), message, esDoc);
        }
        String indexName = ElasticsearchUtils.getIndexName(sensorType, indexPostfix, configurations);
        IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName, sensorType + "_doc");
        indexRequestBuilder = indexRequestBuilder.setSource(esDoc.toJSONString());
        String guid = (String) esDoc.get(Constants.GUID);
        if (guid != null) {
            indexRequestBuilder.setId(guid);
        }
        Object ts = esDoc.get("timestamp");
        if (ts != null) {
            indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString());
        }
        bulkRequest.add(indexRequestBuilder);
    }
    BulkResponse bulkResponse = bulkRequest.execute().actionGet();
    return buildWriteReponse(tuples, bulkResponse);
}

Example 73 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project metron by apache.

the class ElasticSearchComponent method add.

public BulkResponse add(String indexName, String sensorType, Iterable<String> docs) throws IOException {
    BulkRequestBuilder bulkRequest = getClient().prepareBulk();
    for (String doc : docs) {
        IndexRequestBuilder indexRequestBuilder = getClient().prepareIndex(indexName, sensorType + "_doc");
        indexRequestBuilder = indexRequestBuilder.setSource(doc);
        Map<String, Object> esDoc = JSONUtils.INSTANCE.load(doc, JSONUtils.MAP_SUPPLIER);
        indexRequestBuilder.setId((String) esDoc.get(Constants.GUID));
        Object ts = esDoc.get("timestamp");
        if (ts != null) {
            indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString());
        }
        bulkRequest.add(indexRequestBuilder);
    }
    BulkResponse response = bulkRequest.execute().actionGet();
    if (response.hasFailures()) {
        throw new IOException(response.buildFailureMessage());
    }
    return response;
}

Example 74 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project metron by apache.

the class ElasticsearchWriterTest method testSuccessAndFailure.

@Test
public void testSuccessAndFailure() throws Exception {
    Tuple tuple1 = mock(Tuple.class);
    Tuple tuple2 = mock(Tuple.class);
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(true);
    Exception e = new IllegalStateException("Cause");
    BulkItemResponse itemResponse = buildBulkItemFailure(e);
    BulkItemResponse itemResponse2 = mock(BulkItemResponse.class);
    when(itemResponse2.isFailed()).thenReturn(false);
    when(response.iterator()).thenReturn(ImmutableList.of(itemResponse, itemResponse2).iterator());
    BulkWriterResponse expected = new BulkWriterResponse();
    expected.addError(e, tuple1);
    expected.addSuccess(tuple2);
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    BulkWriterResponse actual = esWriter.buildWriteReponse(ImmutableList.of(tuple1, tuple2), response);
    assertEquals("Response should have one error and one success", expected, actual);
}

Example 75 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project metron by apache.

the class ElasticsearchWriterTest method testSingleSuccesses.

@Test
public void testSingleSuccesses() throws Exception {
    Tuple tuple1 = mock(Tuple.class);
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(false);
    BulkWriterResponse expected = new BulkWriterResponse();
    expected.addSuccess(tuple1);
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    BulkWriterResponse actual = esWriter.buildWriteReponse(ImmutableList.of(tuple1), response);
    assertEquals("Response should have no errors and single success", expected, actual);
}