Examples with BulkResponse org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 76 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project metron by apache.

the class ElasticsearchWriterTest method testTwoSameFailure.

@Test
public void testTwoSameFailure() throws Exception {
    Tuple tuple1 = mock(Tuple.class);
    Tuple tuple2 = mock(Tuple.class);
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(true);
    Exception e = new IllegalStateException();
    BulkItemResponse itemResponse = buildBulkItemFailure(e);
    BulkItemResponse itemResponse2 = buildBulkItemFailure(e);
    when(response.iterator()).thenReturn(ImmutableList.of(itemResponse, itemResponse2).iterator());
    BulkWriterResponse expected = new BulkWriterResponse();
    expected.addError(e, tuple1);
    expected.addError(e, tuple2);
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    BulkWriterResponse actual = esWriter.buildWriteReponse(ImmutableList.of(tuple1, tuple2), response);
    assertEquals("Response should have two errors and no successes", expected, actual);
    // Ensure the errors actually get collapsed together
    Map<Throwable, Collection<Tuple>> actualErrors = actual.getErrors();
    HashMap<Throwable, Collection<Tuple>> expectedErrors = new HashMap<>();
    expectedErrors.put(e, ImmutableList.of(tuple1, tuple2));
    assertEquals("Errors should have collapsed together", expectedErrors, actualErrors);
}

Example 77 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project metron by apache.

the class ElasticsearchWriterTest method testTwoDifferentFailure.

@Test
public void testTwoDifferentFailure() throws Exception {
    Tuple tuple1 = mock(Tuple.class);
    Tuple tuple2 = mock(Tuple.class);
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(true);
    Exception e = new IllegalStateException("Cause");
    Exception e2 = new IllegalStateException("Different Cause");
    BulkItemResponse itemResponse = buildBulkItemFailure(e);
    BulkItemResponse itemResponse2 = buildBulkItemFailure(e2);
    when(response.iterator()).thenReturn(ImmutableList.of(itemResponse, itemResponse2).iterator());
    BulkWriterResponse expected = new BulkWriterResponse();
    expected.addError(e, tuple1);
    expected.addError(e2, tuple2);
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    BulkWriterResponse actual = esWriter.buildWriteReponse(ImmutableList.of(tuple1, tuple2), response);
    assertEquals("Response should have two errors and no successes", expected, actual);
    // Ensure the errors did not get collapsed together
    Map<Throwable, Collection<Tuple>> actualErrors = actual.getErrors();
    HashMap<Throwable, Collection<Tuple>> expectedErrors = new HashMap<>();
    expectedErrors.put(e, ImmutableList.of(tuple1));
    expectedErrors.put(e2, ImmutableList.of(tuple2));
    assertEquals("Errors should not have collapsed together", expectedErrors, actualErrors);
}

Example 78 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project uavstack by uavorg.

the class NewLogDataMessageHandler method storeToES.

/**
 * storeToES
 *
 * @param msg
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
private void storeToES(Message msg) {
    String data = msg.getParam(getMsgTypeName());
    List<String> array = JSONHelper.toObjectArray(data, String.class);
    for (String mdfStr : array) {
        // 反序列化为MonitorDataFrame
        MonitorDataFrame mdf = new MonitorDataFrame(mdfStr);
        /**
         * 获取IP和端口，这样唯一性的标识
         */
        String ipport = mdf.getIP();
        String appurl = mdf.getExt("appurl");
        if (!StringHelper.isEmpty(appurl)) {
            ipport = appurl.split("/")[2];
        }
        Map<String, List<Map>> frames = mdf.getDatas();
        for (String appid : frames.keySet()) {
            BulkRequestBuilder bulkRequest = client.getClient().prepareBulk();
            List<Map> applogs = frames.get(appid);
            for (Map applog : applogs) {
                List<Map> instances = (List<Map>) applog.get("Instances");
                for (Map logData : instances) {
                    // push to ES BulkRequest
                    pushLogLineToBulkRequest(mdf, appid, ipport, bulkRequest, logData);
                }
            }
            BulkResponse bulkResponse = bulkRequest.get();
            if (bulkResponse.hasFailures()) {
                log.err(this, "INSERT App[" + appid + "][" + mdf.getIP() + "] on " + mdf.getServerId() + " Log Data to ES FAIL: " + bulkResponse.buildFailureMessage());
            }
        }
    }
}

Example 79 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project uavstack by uavorg.

the class InvokeChainDataCollectHandler method handle.

@Override
public void handle(CollectDataFrame frame) {
    if (this.log.isDebugEnable()) {
        this.log.debug(this, frame.toJSONString());
    }
    String appUUID = frame.getTarget();
    // cm.beginBatch();
    BulkRequestBuilder bulkRequest = client.getClient().prepareBulk();
    for (Line line : frame.getLines()) {
        String content = line.getContent();
        Span span = new Span(content);
        pushLatestIVCDataToCache(appUUID, span);
        pushSpanToBulkRequest(appUUID, frame.getAppgroup(), span, bulkRequest);
    }
    // cm.submitBatch();
    BulkResponse bulkResponse = bulkRequest.get();
    if (bulkResponse.hasFailures()) {
        log.err(this, "INSERT InvokeChain Data to ES FAIL: " + bulkResponse.buildFailureMessage());
    }
}

Example 80 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project ecs-dashboard by carone1.

the class ElasticBucketOwnerDAO method insert.

@Override
public void insert(List<BucketOwner> bucketOwners, Date collectionTime) {
    if (bucketOwners != null && !bucketOwners.isEmpty()) {
        BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
        // Generate JSON for object buckets info
        for (BucketOwner bucketOwner : bucketOwners) {
            XContentBuilder objectBucketBuilder = toJsonFormat(bucketOwner, collectionTime);
            IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(bucketownerIndexDayName).setType(BUCKET_OWNER_INDEX_TYPE).setSource(objectBucketBuilder);
            requestBuilder.add(request);
        }
        BulkResponse bulkResponse = requestBuilder.execute().actionGet();
        int items = bulkResponse.getItems().length;
        LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in ElasticSearch" + "index: " + bucketownerIndexDayName + " index type: " + BUCKET_OWNER_INDEX_TYPE);
        if (bulkResponse.hasFailures()) {
            LOGGER.error("Failures occured while items in ElasticSearch " + "index: " + bucketownerIndexDayName + " index type: " + BUCKET_OWNER_INDEX_TYPE);
        }
    }
}