Examples with DataCollector com.creditease.uav.collect.client.collectdata.DataCollector used on opensource projects

Example 1 with DataCollector

use of com.creditease.uav.collect.client.collectdata.DataCollector in project uavstack by uavorg.

the class CopyOfProcessOfLogagent method sendLogDataBatch.

@SuppressWarnings("unchecked")
protected void sendLogDataBatch(Map<TailFile, List<Map>> serverlogs) {
    DataCollector dc = (DataCollector) ConfigurationManager.getInstance().getComponent("collectclient", DataCollector.class.getName());
    for (Entry<TailFile, List<Map>> en : serverlogs.entrySet()) {
        TailFile tf = en.getKey();
        List<Map> data = en.getValue();
        CollectDataFrame frame = new CollectDataFrame(tf.getServerId(), tf.getAppId(), tf.getPath());
        // 给日志设置应用组
        frame.setAppgroup(System.getProperty("JAppGroup"));
        for (Map<String, String> m : data) {
            frame.append(Integer.parseInt(m.get("_lnum")), m.get("content"), Long.parseLong(m.get("_timestamp")));
        }
        dc.submit(frame);
    }
}

Example 2 with DataCollector

use of com.creditease.uav.collect.client.collectdata.DataCollector in project uavstack by uavorg.

the class ReliableTaildirEventReader method updatelog.

public void updatelog(Map<String, LogPatternInfo> filePaths) {
    for (Entry<String, LogPatternInfo> e : filePaths.entrySet()) {
        LogPatternInfo logPatternInfo = e.getValue();
        DataCollector dc = (DataCollector) ConfigurationManager.getInstance().getComponent("collectclient", DataCollector.class.getName());
        LogPatternInfo logPatternInfoTemp = dc.getLatestLogProfileDataMap().get(logPatternInfo.getAppUUID(), logPatternInfo.getUUID());
        List<File> list = getMatchFiles(logPatternInfo.getParentDir(), logPatternInfo.getLogRegxPattern());
        if (!list.isEmpty()) {
            logPatternInfoTemp.setFlag(StateFlag.EXIST);
        } else {
            logPatternInfoTemp.setFlag(StateFlag.EXIST_UNKOWN);
        }
        if (list.isEmpty() && !logPatternInfo.getParentDir().isDirectory()) {
        // notify
        /*
                 * String title = NetworkHelper.getLocalIP() + "在" + logPatternInfo.getParentDir() + "下没有符合日志文件匹配规则[" +
                 * logPatternInfo.getLogRegxPattern() + "]的日志文件。"; String content =
                 * "失败原因：1）日志文件匹配规则配置错误，所以不能定位日志文件。2）日志文件的命名已经改变，但没有修过日志文件匹配规则。";
                 * 
                 * logger.warn(this, title);
                 * 
                 * AgentFeatureComponent afc = (AgentFeatureComponent) ConfigurationManager.getInstance()
                 * .getComponent("logagent");
                 * 
                 * NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogNotExist, title, content);
                 * event.addArg("serverid", logPatternInfo.getServId()); event.addArg("appid",
                 * logPatternInfo.getAppId()); afc.putNotificationEvent(event);
                 */
        }
        // <R=filepath,C=logPatternInfo,V=
        tailFileTable.put(logPatternInfo.getAbsolutePath(), logPatternInfo);
    }
// logger.info(this, "update log table --" + tailFileTable.toString());
}

Example 3 with DataCollector

use of com.creditease.uav.collect.client.collectdata.DataCollector in project uavstack by uavorg.

the class ReliableTaildirEventReader method updateTailFiles.

/**
 * Update tailFiles mapping if a new file is created or appends are detected to the existing file.
 */
public List<Long> updateTailFiles(boolean skipToEnd) throws IOException {
    updateTime = System.currentTimeMillis();
    List<Long> updatedInodes = new ArrayList<>();
    String serverid = null;
    String appid = null;
    String logid = null;
    for (Entry<String, LogPatternInfo> cell : tailFileTable.asMap().entrySet()) {
        // cell<serverid--appid--logid, logpath, logname>
        // 
        Map<String, String> headers = headerTable.row(cell.getKey());
        LogPatternInfo logPatternInfo = cell.getValue();
        // 文件父路径
        File parentDir = logPatternInfo.getParentDir();
        // 编译后的文件名
        Pattern fileNamePattern = logPatternInfo.getLogRegxPattern();
        serverid = logPatternInfo.getServId();
        appid = logPatternInfo.getAppId();
        logid = logPatternInfo.getLogParttern();
        boolean unsplit = logPatternInfo.isUnsplit();
        List<File> files = getMatchFiles(parentDir, fileNamePattern);
        DataCollector dc = (DataCollector) ConfigurationManager.getInstance().getComponent("collectclient", DataCollector.class.getName());
        LogPatternInfo logPatternInfo2 = dc.getLatestLogProfileDataMap().get(logPatternInfo.getAppUUID(), logPatternInfo.getUUID());
        if (!files.isEmpty()) {
            // modify status UNKNOWN to EXISTS
            if (logPatternInfo2 != null) {
                logPatternInfo2.setFlag(StateFlag.EXIST);
            }
        } else if (logPatternInfo2.getFlag() == StateFlag.EXIST) {
            logPatternInfo2.setFlag(StateFlag.EXIST_UNKOWN);
            String title = NetworkHelper.getLocalIP() + "曾经在" + logPatternInfo.getParentDir() + "符合日志文件匹配规则[" + logPatternInfo.getLogRegxPattern() + "]的日志文件消失了。";
            // String content = "失败原因：1）错误删除了这些日志文件。2）修改了日志文件名称，且新名称不符合日志文件匹配规则[" +
            // logPatternInfo.getLogRegxPattern()
            // + "]。";
            log.warn(this, title);
        // AgentFeatureComponent afc = (AgentFeatureComponent) ConfigurationManager.getInstance()
        // .getComponent("logagent");
        // 
        // NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogNotExist, title, content);
        // event.addArg("serverid", logPatternInfo.getServId());
        // event.addArg("appid", logPatternInfo.getAppId());
        // afc.putNotificationEvent(event);
        }
        for (File f : files) {
            long inode = getInode(f);
            TailFile tf = tailFiles.get(inode);
            if (tf == null || !tf.getPath().equals(f.getAbsolutePath())) {
                // 第一次读取从头开始读
                long startPos = skipToEnd ? f.length() : 0;
                // how to get line's number ?
                long startNum = 0;
                // try to get pos form position file
                if (maybeReloadMap.containsKey(inode)) {
                    startPos = maybeReloadMap.get(inode)[0];
                    startNum = maybeReloadMap.get(inode)[1];
                }
                tf = openFile(serverid, appid, logid, f, headers, inode, startPos, startNum, unsplit);
            } else {
                boolean updated = tf.getLastUpdated() < f.lastModified();
                if (updated) {
                    if (tf.getRaf() == null) {
                        // 获取文件的读取手柄
                        tf = openFile(serverid, appid, logid, f, headers, inode, tf.getPos(), tf.getNum(), unsplit);
                    }
                    if (f.length() < tf.getPos()) {
                        // 文件的长度小于上次读取的指针说明文件内容被删除了，改成从0读取
                        log.info(this, "Pos " + tf.getPos() + " is larger than file size! " + "Restarting from pos 0, file: " + tf.getPath() + ", inode: " + inode);
                        tf.updatePos(tf.getPath(), inode, 0, 0);
                    }
                }
                // 设置是否需要监控指标
                tf.setNeedTail(updated);
            }
            tailFiles.put(inode, tf);
            updatedInodes.add(inode);
        }
    }
    return updatedInodes;
}

Example 4 with DataCollector

use of com.creditease.uav.collect.client.collectdata.DataCollector in project uavstack by uavorg.

the class TailFile method readEvents.

public List<Event> readEvents(int numEvents, boolean backoffWithoutNL, boolean addByteOffset, boolean mutiFlag) throws IOException {
    List<Event> events = new LinkedList<>();
    // Fix:computer the eventSize, if the size overflow Max size, break the numEvents loop
    int sumEventsLength = mutiFlag ? 0 : getCurrentSumEventsLength();
    if (unsplit) {
        numEvents = Integer.MAX_VALUE;
    }
    for (int i = 0; i < numEvents; i++) {
        Event event = readEvent(backoffWithoutNL, addByteOffset);
        if (event == null) {
            break;
        }
        if (event == Eof.instance()) {
            if (!eofOccur) {
                eofOccur = true;
                DataCollector dc = (DataCollector) ConfigurationManager.getInstance().getComponent("collectclient", DataCollector.class.getName());
                dc.onTaskEof(serverid, appid);
            }
            break;
        }
        // readLineNumber++;
        sumEventsLength += event.getBodyLength();
        if (sumEventsLength > readMaxByte) {
            // logger.err(this, "## overflow Max size ### sumEventsLength ###: " + sumEventsLength);
            break;
        }
        events.add(event);
    }
    currentSumEventsLength = sumEventsLength;
    return events;
}

Example 5 with DataCollector

use of com.creditease.uav.collect.client.collectdata.DataCollector in project uavstack by uavorg.

the class CollectDataAgent method start.

@Override
public void start() {
    // init position file. If file is not exists, create it
    String rootMetaPath = this.getConfigManager().getContext(IConfigurationManager.METADATAPATH) + "collectdata";
    IOHelper.createFolder(rootMetaPath);
    positionFilePath = rootMetaPath + File.separator + POSITION_FILE;
    taskFilePath = rootMetaPath + File.separator + TASK_FILE;
    initFiles(new String[] { positionFilePath, taskFilePath });
    // register CollectNodeOperAction
    IActionEngine engine = this.getActionEngineMgr().getActionEngine("NodeOperActionEngine");
    new CollectNodeOperAction("collectdata", feature, engine);
    // init DataCollector
    dc = new DataCollector(DataCollector.class.getName(), feature);
    List<CollectTask> tasks = loadTasks();
    dc.init(tasks);
    dc.loadPositions(IOHelper.readTxtFile(positionFilePath, "UTF-8"));
    // start data scaner
    long scanInterval = DataConvertHelper.toLong(getConfigManager().getFeatureConfiguration(feature, "interval"), 3000L);
    FileScanScheduler scanner = new FileScanScheduler(FileScanScheduler.class.getName(), feature);
    getTimerWorkManager().scheduleWork(FileScanScheduler.class.getName(), scanner, 0, scanInterval);
    // write position file scheduler
    long writePosDelay = DataConvertHelper.toLong(getConfigManager().getFeatureConfiguration(feature, "writeposdelay"), 5000L);
    long writePosInterval = DataConvertHelper.toLong(getConfigManager().getFeatureConfiguration(feature, "writeposinterval"), 3000L);
    PositionPersistence position = new PositionPersistence(PositionPersistence.class.getName(), feature);
    getTimerWorkManager().scheduleWork(PositionPersistence.class.getName(), position, writePosDelay, writePosInterval);
    // check idle file scheduler
    long idleCheckerDelay = DataConvertHelper.toLong(getConfigManager().getFeatureConfiguration(feature, "idlecheckerdelay"), 120000L);
    long idleCheckerInterval = DataConvertHelper.toLong(getConfigManager().getFeatureConfiguration(feature, "idlecheckerinterval"), 5000L);
    IdleFileChecker idleChecker = new IdleFileChecker(IdleFileChecker.class.getName(), feature);
    getTimerWorkManager().scheduleWork(IdleFileChecker.class.getName(), idleChecker, idleCheckerDelay, idleCheckerInterval);
    log.info(this, "CollectDataAgent Started. Config: interval=" + scanInterval);
}