Examples with BulkResponse org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 96 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project uavstack by uavorg.

the class NewLogDataMessageHandler method storeToES.

/**
 * storeToES
 *
 * @param msg
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
private void storeToES(Message msg) {
    String data = msg.getParam(getMsgTypeName());
    List<String> array = JSONHelper.toObjectArray(data, String.class);
    for (String mdfStr : array) {
        // 反序列化为MonitorDataFrame
        MonitorDataFrame mdf = new MonitorDataFrame(mdfStr);
        /**
         * 获取IP和端口，这样唯一性的标识
         */
        String ipport = mdf.getIP();
        String appurl = mdf.getExt("appurl");
        if (!StringHelper.isEmpty(appurl)) {
            ipport = appurl.split("/")[2];
        }
        Map<String, List<Map>> frames = mdf.getDatas();
        for (String appid : frames.keySet()) {
            BulkRequestBuilder bulkRequest = client.getClient().prepareBulk();
            List<Map> applogs = frames.get(appid);
            for (Map applog : applogs) {
                List<Map> instances = (List<Map>) applog.get("Instances");
                for (Map logData : instances) {
                    // push to ES BulkRequest
                    pushLogLineToBulkRequest(mdf, appid, ipport, bulkRequest, logData);
                }
            }
            BulkResponse bulkResponse = bulkRequest.get();
            if (bulkResponse.hasFailures()) {
                log.err(this, "INSERT App[" + appid + "][" + mdf.getIP() + "] on " + mdf.getServerId() + " Log Data to ES FAIL: " + bulkResponse.buildFailureMessage());
            }
        }
    }
}

Example 97 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project uavstack by uavorg.

the class InvokeChainDataCollectHandler method handle.

@Override
public void handle(CollectDataFrame frame) {
    if (this.log.isDebugEnable()) {
        this.log.debug(this, frame.toJSONString());
    }
    String appUUID = frame.getTarget();
    // cm.beginBatch();
    BulkRequestBuilder bulkRequest = client.getClient().prepareBulk();
    for (Line line : frame.getLines()) {
        try {
            String content = line.getContent();
            Span span = new Span(content);
            pushLatestIVCDataToCache(appUUID, span);
            pushSpanToBulkRequest(appUUID, frame.getAppgroup(), span, bulkRequest);
        } catch (Exception e) {
            this.log.err(this, "unsupported ivc content :" + line.getContent(), e);
        }
    }
    // cm.submitBatch();
    BulkResponse bulkResponse = bulkRequest.get();
    if (bulkResponse.hasFailures()) {
        log.err(this, "INSERT InvokeChain Data to ES FAIL: " + bulkResponse.buildFailureMessage());
    }
}

Example 98 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project uavstack by uavorg.

the class DoTestTransportHookProxy method main.

@SuppressWarnings("unchecked")
public static void main(String[] args) throws IOException {
    ConsoleLogger cl = new ConsoleLogger("test");
    cl.setDebugable(true);
    UAVServer.instance().setLog(cl);
    UAVServer.instance().putServerInfo(CaptureConstants.INFO_APPSERVER_VENDOR, ServerVendor.TOMCAT);
    MOFAgent.mofContext.put("org.uavstack.mof.ext.clsloader", Thread.currentThread().getContextClassLoader());
    TransportHookProxy p = new TransportHookProxy("test", Collections.emptyMap());
    p.doProxyInstall(null, "testApp");
    String[] esAddrs = { "127.0.0.1:9300" };
    String clusterName = "";
    String index = "esindex";
    String type = "String";
    String alias = "alias";
    Boolean result;
    Settings settings = Settings.EMPTY;
    if (!StringHelper.isEmpty(clusterName)) {
        settings = Settings.builder().put("cluster.name", clusterName).build();
    }
    TransportClient client = new PreBuiltTransportClient(settings);
    for (String esAddr : esAddrs) {
        String[] ipport = esAddr.split(":");
        client.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(ipport[0], DataConvertHelper.toInt(ipport[1], 9300))));
    }
    result = client.admin().indices().exists(new IndicesExistsRequest(index)).actionGet().isExists();
    if (result) {
        result = client.admin().indices().delete(new DeleteIndexRequest(index)).actionGet().isAcknowledged();
    }
    client.admin().indices().create(new CreateIndexRequest(index)).actionGet().isAcknowledged();
    client.admin().indices().typesExists(new TypesExistsRequest(new String[] { index }, type)).actionGet().isExists();
    client.admin().indices().prepareAliases().addAlias(index, alias).get().isAcknowledged();
    client.admin().indices().prepareAliases().removeAlias(index, alias).get().isAcknowledged();
    client.prepareSearch(index).setSearchType(SearchType.DFS_QUERY_THEN_FETCH).get(TimeValue.timeValueMillis(15000));
    Map<String, Object> m = new HashMap<String, Object>();
    m.put("user", "kimchy");
    m.put("postDate", new Date());
    m.put("message", "trying out Elasticsearch");
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    bulkRequest.add(client.prepareIndex("twitter", "tweet", "1").setSource(m));
    BulkResponse bulkResponse = bulkRequest.get();
    if (bulkResponse.hasFailures()) {
        System.out.println("Failed");
    }
    client.close();
}

Example 99 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project topcom-cloud by 545314690.

the class ElasticSearchService method bulkInsertData.

/**
 * 功能描述：批量插入数据
 * @param index 索引名
 * @param type 类型
 * @param data (_id 主键, json 数据)
 */
public BulkResponse bulkInsertData(String index, String type, Map<String, String> data) {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    data.forEach((param1, param2) -> {
        bulkRequest.add(client.prepareIndex(index, type, param1).setSource(param2));
    });
    BulkResponse bulkResponse = bulkRequest.get();
    return bulkResponse;
}

Example 100 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project topcom-cloud by 545314690.

the class ElasticSearchService method bulkInsertData.

/**
 * 功能描述：批量插入数据
 * @param index 索引名
 * @param type 类型
 * @param jsonList 批量数据
 */
public void bulkInsertData(String index, String type, List<String> jsonList) {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    jsonList.forEach(item -> {
        bulkRequest.add(client.prepareIndex(index, type).setSource(item));
    });
    BulkResponse bulkResponse = bulkRequest.get();
}