Examples with MonitorDataFrame com.creditease.agent.monitor.api.MonitorDataFrame used on opensource projects

Example 16 with MonitorDataFrame

use of com.creditease.agent.monitor.api.MonitorDataFrame in project uavstack by uavorg.

the class MonitorDataAdapter method prepareInsertObj.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public Object prepareInsertObj(DataStoreMsg msg, DataStoreConnection connection) {
    boolean isDataStoreOK = (this.dataSource.getAvalibleAddressEntry() == null) ? false : true;
    // OpenTSDB points list
    List<Map<String, ?>> points = new ArrayList<Map<String, ?>>();
    CacheManager cm = (CacheManager) ConfigurationManager.getInstance().getComponent("healthmanager", "HMCacheManager");
    int cacheTime = DataConvertHelper.toInt(ConfigurationManager.getInstance().getFeatureConfiguration("healthmanager", "MT_Monitor.ds.cachetime"), 1);
    /**
     * 消息message传输过来的MDF是一个列表包含多个MDF的json字符串
     */
    // parse MDF list
    String monitorDataFramesStr = (String) msg.get(MonitorDataFrame.MessageType.Monitor.toString());
    List<String> monitorDataFrames = JSONHelper.toObjectArray(monitorDataFramesStr, String.class);
    cm.beginBatch();
    for (String mdfStr : monitorDataFrames) {
        // 反序列化为MonitorDataFrame
        MonitorDataFrame mdf = new MonitorDataFrame(mdfStr);
        // get all frames
        Map<String, List<Map>> frames = mdf.getDatas();
        for (String monitorId : frames.keySet()) {
            List<Map> monitorDatas = frames.get(monitorId);
            for (Map monitorData : monitorDatas) {
                String meId = (String) monitorData.get("MEId");
                List<Map> instances = (List<Map>) monitorData.get("Instances");
                for (Map instance : instances) {
                    // get instance id
                    String instanceId = (String) instance.get("id");
                    // get instance fields
                    Map<String, Object> fields = (Map<String, Object>) instance.get("values");
                    /**
                     * Step 1: put the inst's metric data into cache
                     */
                    String encodedInstId = DataStoreHelper.encodeForOpenTSDB(instanceId);
                    String mdCacheKey = getMDCacheKey(mdf, encodedInstId, meId);
                    Map<String, Object> fieldValues = new HashMap<String, Object>();
                    fieldValues.put("data", fields);
                    fieldValues.put("time", mdf.getTimeFlag());
                    cm.rpush(HealthManagerConstants.STORE_REGION_UAV, mdCacheKey, JSONHelper.toString(fieldValues));
                    cm.expire(HealthManagerConstants.STORE_REGION_UAV, mdCacheKey, cacheTime, TimeUnit.MINUTES);
                    /**
                     * FAST Failure: if there is no available address, we may not go to DataStore part
                     */
                    if (isDataStoreOK == false) {
                        if (log.isDebugEnable() == true) {
                            log.debug(this, "Skip MonitorData INSERT as no available address");
                        }
                        continue;
                    }
                    /**
                     * Step 2: prepare the inst's metic data for opentsdb
                     */
                    for (String key : fields.keySet()) {
                        Map<String, Object> point = new LinkedHashMap<String, Object>();
                        String mkey = key;
                        /**
                         * NOTE: RC=return code, they are dynamic tags, so make the metric key as "XX.rc" add a new
                         * tag "ptag" as the last tag
                         */
                        String plusTag = null;
                        if (key.indexOf("RC") == 0) {
                            mkey = "RC";
                            plusTag = key;
                        } else if (key.indexOf("AC") == 0) {
                            mkey = "AC";
                            plusTag = key;
                        } else if (key.indexOf("EXT") == 0) {
                            mkey = "EXT";
                            plusTag = key;
                        }
                        /**
                         * WRITE DATA Schema Sample:
                         *
                         * {
                         *
                         * metric: urlResp.RC,
                         *
                         * timestamp: 1463816026510,
                         *
                         * value: 200,
                         *
                         * tags:[
                         *
                         * ip: 127.0.0.1,
                         *
                         * pgid:
                         * E:/UAVIDE/tomcat/apache-tomcat-7.0.65---E:/UAVIDE/defaultworkspace/.metadata/.plugins/org
                         * .eclipse.wst.server.core/tmp0,
                         *
                         * instid: http://127.0.0.1:8080/com.creditease.uav.console/rs/godeye/profile/q/cache,
                         *
                         * ptag(optional): RC400
                         *
                         * ]
                         *
                         * }
                         */
                        point.put("metric", meId + "." + mkey);
                        point.put("timestamp", mdf.getTimeFlag());
                        point.put("value", fields.get(key));
                        Map<String, String> tags = new LinkedHashMap<>();
                        // IP
                        tags.put("ip", DataStoreHelper.encodeForOpenTSDB(mdf.getIP()));
                        // Program Id: Tomcat Server Profile Or Other Service Program Profile
                        tags.put("pgid", DataStoreHelper.encodeForOpenTSDB(mdf.getServerId()));
                        // Instance Id: May be A Tomcat Server, A Tomcat App, A Tomcat App URL or Process Id
                        tags.put("instid", encodedInstId);
                        if (plusTag != null) {
                            tags.put("ptag", DataStoreHelper.encodeForOpenTSDB(plusTag));
                        }
                        /**
                         * note: not store host as no use for tags
                         */
                        // tags.put("host", mdf.getHost());
                        point.put("tags", tags);
                        points.add(point);
                    }
                }
            }
        }
    }
    cm.submitBatch();
    return points;
}

Example 17 with MonitorDataFrame

use of com.creditease.agent.monitor.api.MonitorDataFrame in project uavstack by uavorg.

the class LineNumberWriter method writeLogToFile.

@SuppressWarnings({ "unchecked", "rawtypes" })
public void writeLogToFile(Object... data) {
    List<MonitorDataFrame> mdflist = (List<MonitorDataFrame>) data[0];
    for (MonitorDataFrame mdf : mdflist) {
        List<Map> instances = mdf.getElemInstances("com.creditease.uav.logProducer", "log");
        for (Map instance : instances) {
            String logFileName = (String) instance.get("id");
            int mapIndex = obtainMapIndex(logFileName);
            Map<String, Object> map = list.get(mapIndex);
            if (map.isEmpty()) {
                map.put("fileName", logFileName);
                map.put("lineNum", 0);
            }
            File file = new File(logFileName);
            ISystemLogger logger = null;
            if (!logMap.containsKey(logFileName)) {
                logger = SystemLogger.getTestlogger(file.getName() + "-logger", file.getName() + ".%g.%u.log", null, true, 100 * 1024 * 1024, 1);
                logMap.put(logFileName, logger);
            }
            logger = logMap.get(logFileName);
            Map valuesMap = (Map) instance.get("values");
            String values = JSON.toJSONString(valuesMap);
            JSONObject contentObeject = JSONObject.parseObject(values);
            JSONArray array = contentObeject.getJSONArray("content");
            int logLineNumber = 0;
            for (int i = 0; i < array.size(); i++) {
                JSONObject contentObejct = array.getJSONObject(i);
                String content = contentObejct.getString("content");
                logger.trace(this, content, true, data);
                logLineNumber++;
            }
            map.put("lineNum", (int) map.get("lineNum") + logLineNumber);
        }
    }
}

Example 18 with MonitorDataFrame

use of com.creditease.agent.monitor.api.MonitorDataFrame in project uavstack by uavorg.

the class NewLogDataMessageHandler method storeToES.

/**
 * storeToES
 *
 * @param msg
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
private void storeToES(Message msg) {
    String data = msg.getParam(getMsgTypeName());
    List<String> array = JSONHelper.toObjectArray(data, String.class);
    for (String mdfStr : array) {
        // 反序列化为MonitorDataFrame
        MonitorDataFrame mdf = new MonitorDataFrame(mdfStr);
        /**
         * 获取IP和端口，这样唯一性的标识
         */
        String ipport = mdf.getIP();
        String appurl = mdf.getExt("appurl");
        if (!StringHelper.isEmpty(appurl)) {
            ipport = appurl.split("/")[2];
        }
        Map<String, List<Map>> frames = mdf.getDatas();
        for (String appid : frames.keySet()) {
            BulkRequestBuilder bulkRequest = client.getClient().prepareBulk();
            List<Map> applogs = frames.get(appid);
            for (Map applog : applogs) {
                List<Map> instances = (List<Map>) applog.get("Instances");
                for (Map logData : instances) {
                    // push to ES BulkRequest
                    pushLogLineToBulkRequest(mdf, appid, ipport, bulkRequest, logData);
                }
            }
            BulkResponse bulkResponse = bulkRequest.get();
            if (bulkResponse.hasFailures()) {
                log.err(this, "INSERT App[" + appid + "][" + mdf.getIP() + "] on " + mdf.getServerId() + " Log Data to ES FAIL: " + bulkResponse.buildFailureMessage());
            }
        }
    }
}