Examples with BulkResponse org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 81 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project ecs-dashboard by carone1.

the class ElasticS3ObjectDAO method insert.

/**
 * {@inheritDoc}
 */
@Override
public void insert(QueryObjectsResult queryObjectsResult, String namespace, String bucketName, Date collectionTime) {
    if (queryObjectsResult == null || queryObjectsResult.getObjects() == null || queryObjectsResult.getObjects().isEmpty()) {
        // nothing to insert
        return;
    }
    BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
    // Generate JSON for object buckets info
    for (QueryObject queryObject : queryObjectsResult.getObjects()) {
        XContentBuilder s3ObjectBuilder = toJsonFormat(queryObject, namespace, bucketName, collectionTime);
        IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(s3ObjectIndexDayName).setType(S3_OBJECT_INDEX_TYPE).setSource(s3ObjectBuilder);
        requestBuilder.add(request);
    }
    BulkResponse bulkResponse = requestBuilder.execute().actionGet();
    int items = bulkResponse.getItems().length;
    LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in Elasticsearch " + "index: " + s3ObjectIndexDayName + " index type: " + S3_OBJECT_INDEX_TYPE);
    if (bulkResponse.hasFailures()) {
        LOGGER.error("Failure(s) occured while items in Elasticsearch " + "index: " + s3ObjectIndexDayName + " index type: " + S3_OBJECT_INDEX_TYPE);
    }
}

Example 82 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project ecs-dashboard by carone1.

the class ElasticS3ObjectDAO method insert.

/**
 * {@inheritDoc}
 */
@Override
public void insert(ListVersionsResult listVersionsResult, String namespace, String bucketName, Date collectionTime) {
    if (listVersionsResult == null || listVersionsResult.getVersions() == null || listVersionsResult.getVersions().isEmpty()) {
        // nothing to insert
        return;
    }
    BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
    // Generate JSON for object version info
    for (AbstractVersion abstractVersion : listVersionsResult.getVersions()) {
        if (abstractVersion instanceof Version) {
            XContentBuilder s3ObjectVersionBuilder = toJsonFormat((Version) abstractVersion, namespace, bucketName, collectionTime);
            IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(s3ObjectVersionIndexDayName).setType(S3_OBJECT_VERSION_INDEX_TYPE).setSource(s3ObjectVersionBuilder);
            requestBuilder.add(request);
        } else if (abstractVersion instanceof DeleteMarker) {
            XContentBuilder s3ObjectVersionBuilder = toJsonFormat((DeleteMarker) abstractVersion, namespace, bucketName, collectionTime);
            IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(s3ObjectVersionIndexDayName).setType(S3_OBJECT_VERSION_INDEX_TYPE).setSource(s3ObjectVersionBuilder);
            requestBuilder.add(request);
        }
    }
    BulkResponse bulkResponse = requestBuilder.execute().actionGet();
    int items = bulkResponse.getItems().length;
    LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in Elasticsearch " + "index: " + s3ObjectVersionIndexDayName + " index type: " + S3_OBJECT_VERSION_INDEX_TYPE);
    if (bulkResponse.hasFailures()) {
        LOGGER.error("Failure(s) occured while items in Elasticsearch " + "index: " + s3ObjectVersionIndexDayName + " index type: " + S3_OBJECT_VERSION_INDEX_TYPE);
    }
}

Example 83 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project ecs-dashboard by carone1.

the class ElasticVdcDetailDAO method insert.

@Override
public void insert(VdcDetails vdcDetails, Date collectionTime) {
    if (vdcDetails == null || vdcDetails.getVdcDetails() == null || vdcDetails.getVdcDetails().isEmpty()) {
        return;
    }
    BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
    // Generate JSON for object buckets info
    for (VdcDetail vdcDetail : vdcDetails.getVdcDetails()) {
        XContentBuilder objectBucketBuilder = toJsonFormat(vdcDetail, collectionTime);
        IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(vdcIndexDayName).setType(VDC_INDEX_TYPE).setSource(objectBucketBuilder);
        requestBuilder.add(request);
    }
    BulkResponse bulkResponse = requestBuilder.execute().actionGet();
    int items = bulkResponse.getItems().length;
    LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in ElasticSearch" + "index: " + vdcIndexDayName + " index type: " + VDC_INDEX_TYPE);
    if (bulkResponse.hasFailures()) {
        LOGGER.error("Failures occured while items in ElasticSearch " + "index: " + vdcIndexDayName + " index type: " + VDC_INDEX_TYPE);
    }
}

Example 84 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project incubator-skywalking by apache.

the class BatchEsDAO method batchPersistence.

@Override
public void batchPersistence(List<?> batchCollection) {
    BulkRequestBuilder bulkRequest = getClient().prepareBulk();
    logger.debug("bulk data size: {}", batchCollection.size());
    if (CollectionUtils.isNotEmpty(batchCollection)) {
        batchCollection.forEach(builder -> {
            if (builder instanceof IndexRequestBuilder) {
                bulkRequest.add((IndexRequestBuilder) builder);
            }
            if (builder instanceof UpdateRequestBuilder) {
                bulkRequest.add((UpdateRequestBuilder) builder);
            }
        });
        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
        if (bulkResponse.hasFailures()) {
            logger.error(bulkResponse.buildFailureMessage());
            for (BulkItemResponse itemResponse : bulkResponse.getItems()) {
                logger.error("Bulk request failure, index: {}, id: {}", itemResponse.getIndex(), itemResponse.getId());
            }
        }
    }
}

Example 85 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project kylo by Teradata.

the class IndexElasticSearch method sendToElasticSearch.

private boolean sendToElasticSearch(String json, String hostName, String index, String type, String clusterName, String idField, String categoryName, String feedName) throws Exception {
    final ComponentLog logger = getLog();
    Settings settings = Settings.settingsBuilder().put("cluster.name", clusterName).build();
    Client client = TransportClient.builder().settings(settings).build().addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(hostName), 9300));
    JSONArray array = new JSONArray(json);
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (int i = 0; i < array.length(); i++) {
        JSONObject jsonObj = array.getJSONObject(i);
        String id = null;
        if (idField != null && idField.length() > 0) {
            id = jsonObj.getString(idField);
            logger.debug("Document index id using field " + idField + ": " + id);
        } else if (StringUtils.isNotEmpty(categoryName) && (StringUtils.isNotEmpty(feedName))) {
            String hash = InsecureHashingMessageUtil.getHashMD5(jsonObj.toString());
            if (StringUtils.isNotEmpty(hash)) {
                id = categoryName + "::" + feedName + "::" + hash;
                logger.debug("Document index id using hash: " + id);
            }
        }
        if (StringUtils.isEmpty(id)) {
            id = UUID.randomUUID().toString();
            logger.debug("Document index id auto-generated + " + id);
        }
        jsonObj.put("post_date", String.valueOf(System.currentTimeMillis()));
        bulkRequest.add(client.prepareIndex(index, type, id).setSource(jsonObj.toString()));
    }
    BulkResponse bulkResponse = bulkRequest.get();
    if (bulkResponse.hasFailures()) {
        logger.error("Error occurred while batch updating" + bulkResponse.buildFailureMessage());
        return false;
    }
    return true;
}