Examples with ThroughputStat com.alibaba.otter.shared.common.model.statistics.throughput.ThroughputStat used on opensource projects

Example 6 with ThroughputStat

use of com.alibaba.otter.shared.common.model.statistics.throughput.ThroughputStat in project otter by alibaba.

the class OtterLoaderFactory method sendStat.

private void sendStat(Identity identity) {
    LoadThroughput throughput = loadStatsTracker.getStat(identity);
    Collection<LoadCounter> counters = throughput.getStats();
    Date endTime = new Date();
    // 处理table stat
    long fileSize = 0L;
    long fileCount = 0L;
    long rowSize = 0L;
    long rowCount = 0L;
    long mqSize = 0L;
    long mqCount = 0L;
    List<TableStat> tableStats = new ArrayList<TableStat>();
    for (LoadCounter counter : counters) {
        TableStat stat = new TableStat();
        stat.setPipelineId(identity.getPipelineId());
        stat.setDataMediaPairId(counter.getPairId());
        stat.setFileCount(counter.getFileCount().longValue());
        stat.setFileSize(counter.getFileSize().longValue());
        stat.setInsertCount(counter.getInsertCount().longValue());
        stat.setUpdateCount(counter.getUpdateCount().longValue());
        stat.setDeleteCount(counter.getDeleteCount().longValue());
        stat.setStartTime(new Date(throughput.getStartTime()));
        stat.setEndTime(endTime);
        // 5项中有一项不为空才通知
        if (!(stat.getFileCount().equals(0L) && stat.getFileSize().equals(0L) && stat.getInsertCount().equals(0L) && stat.getDeleteCount().equals(0L) && stat.getUpdateCount().equals(0L))) {
            tableStats.add(stat);
        }
        fileSize += counter.getFileSize().longValue();
        fileCount += counter.getFileCount().longValue();
        rowSize += counter.getRowSize().longValue();
        rowCount += counter.getRowCount().longValue();
        mqSize += counter.getMqSize().longValue();
        mqCount += counter.getMqCount().longValue();
    }
    if (!CollectionUtils.isEmpty(tableStats)) {
        statisticsClientService.sendTableStats(tableStats);
    }
    List<ThroughputStat> throughputStats = new ArrayList<ThroughputStat>();
    if (!(rowCount == 0 && rowSize == 0)) {
        // 处理Throughput stat
        ThroughputStat rowThroughputStat = new ThroughputStat();
        rowThroughputStat.setType(ThroughputType.ROW);
        rowThroughputStat.setPipelineId(identity.getPipelineId());
        rowThroughputStat.setNumber(rowCount);
        rowThroughputStat.setSize(rowSize);
        rowThroughputStat.setStartTime(new Date(throughput.getStartTime()));
        rowThroughputStat.setEndTime(endTime);
        throughputStats.add(rowThroughputStat);
    }
    if (!(fileCount == 0 && fileSize == 0)) {
        ThroughputStat fileThroughputStat = new ThroughputStat();
        fileThroughputStat.setType(ThroughputType.FILE);
        fileThroughputStat.setPipelineId(identity.getPipelineId());
        fileThroughputStat.setNumber(fileCount);
        fileThroughputStat.setSize(fileSize);
        fileThroughputStat.setStartTime(new Date(throughput.getStartTime()));
        fileThroughputStat.setEndTime(endTime);
        throughputStats.add(fileThroughputStat);
    }
    // add by 2012-07-06 for mq loader
    if (!(mqCount == 0 && mqSize == 0)) {
        ThroughputStat mqThroughputStat = new ThroughputStat();
        mqThroughputStat.setType(ThroughputType.MQ);
        mqThroughputStat.setPipelineId(identity.getPipelineId());
        mqThroughputStat.setNumber(mqCount);
        mqThroughputStat.setSize(mqSize);
        mqThroughputStat.setStartTime(new Date(throughput.getStartTime()));
        mqThroughputStat.setEndTime(endTime);
        throughputStats.add(mqThroughputStat);
    }
    if (!CollectionUtils.isEmpty(throughputStats)) {
        statisticsClientService.sendThroughputs(throughputStats);
    }
}

Example 7 with ThroughputStat

use of com.alibaba.otter.shared.common.model.statistics.throughput.ThroughputStat in project otter by alibaba.

the class StatsRemoteServiceImpl method flushThroughputStat.

private void flushThroughputStat() {
    synchronized (throughputStats) {
        Collection<Map<ThroughputType, ThroughputStat>> stats = throughputStats.values();
        for (Map<ThroughputType, ThroughputStat> stat : stats) {
            for (ThroughputStat data : stat.values()) {
                throughputStatService.createOrUpdateThroughput(data);
            }
        }
        throughputStats.clear();
    }
}

Example 8 with ThroughputStat

use of com.alibaba.otter.shared.common.model.statistics.throughput.ThroughputStat in project otter by alibaba.

the class StatsRemoteServiceImpl method onThroughputStat.

public void onThroughputStat(ThroughputStatEvent event) {
    Assert.notNull(event);
    Assert.notNull(event.getStats());
    if (statUnit <= 0) {
        for (ThroughputStat stat : event.getStats()) {
            throughputStatService.createOrUpdateThroughput(stat);
        }
    } else {
        synchronized (throughputStats) {
            for (ThroughputStat stat : event.getStats()) {
                Map<ThroughputType, ThroughputStat> data = throughputStats.get(stat.getPipelineId());
                ThroughputStat old = data.get(stat.getType());
                if (old != null) {
                    //执行合并
                    old.setNumber(stat.getNumber() + old.getNumber());
                    old.setSize(stat.getSize() + old.getSize());
                    if (stat.getEndTime().after(old.getEndTime())) {
                        old.setEndTime(stat.getEndTime());
                    }
                    if (stat.getStartTime().before(old.getStartTime())) {
                        old.setStartTime(stat.getStartTime());
                    }
                } else {
                    data.put(stat.getType(), stat);
                }
            }
        }
    }
}

Example 9 with ThroughputStat

use of com.alibaba.otter.shared.common.model.statistics.throughput.ThroughputStat in project otter by alibaba.

the class PipelineTimeoutRuleMonitor method explore.

@Override
public void explore(List<AlarmRule> rules) {
    if (CollectionUtils.isEmpty(rules)) {
        return;
    }
    Long pipelineId = rules.get(0).getPipelineId();
    ThroughputCondition condition = new ThroughputCondition();
    condition.setPipelineId(pipelineId);
    condition.setType(ThroughputType.ROW);
    ThroughputStat stat = throughputStatService.findThroughputStatByPipelineId(condition);
    long latestSyncTime = 0L;
    if (stat != null && stat.getGmtModified() != null) {
        Date modifiedDate = stat.getGmtModified();
        latestSyncTime = modifiedDate.getTime();
    }
    long now = System.currentTimeMillis();
    long elapsed = now - latestSyncTime;
    boolean flag = false;
    for (AlarmRule rule : rules) {
        flag |= checkTimeout(rule, elapsed);
    }
    if (flag) {
        logRecordAlarm(pipelineId, MonitorName.PIPELINETIMEOUT, String.format(TIME_OUT_MESSAGE, pipelineId, (elapsed / 1000)));
    }
}

Example 10 with ThroughputStat

use of com.alibaba.otter.shared.common.model.statistics.throughput.ThroughputStat in project otter by alibaba.

the class ThroughputStatServiceImpl method listTimelineThroughput.

/**
     * <pre>
     * 列出pipeLineId下，start-end时间段下的throughputStat 
     * 首先从数据库中取出这一段时间所以数据，该数据都是根据end_time倒排序的, 每隔1分钟将这些数据分组
     * </pre>
     */
public Map<Long, ThroughputInfo> listTimelineThroughput(TimelineThroughputCondition condition) {
    Assert.assertNotNull(condition);
    Map<Long, ThroughputInfo> throughputInfos = new LinkedHashMap<Long, ThroughputInfo>();
    List<ThroughputStatDO> throughputStatDOs = throughputDao.listTimelineThroughputStat(condition);
    int size = throughputStatDOs.size();
    int k = size - 1;
    for (Long i = condition.getStart().getTime(); i <= condition.getEnd().getTime(); i += 60 * 1000) {
        ThroughputInfo throughputInfo = new ThroughputInfo();
        List<ThroughputStat> throughputStat = new ArrayList<ThroughputStat>();
        // 取出每个时间点i以内的数据，k是一个游标，每次遍历时前面已经取过了的数据就不用再遍历了
        for (int j = k; j >= 0; --j) {
            if ((i - throughputStatDOs.get(j).getEndTime().getTime() <= 60 * 1000) && (i - throughputStatDOs.get(j).getEndTime().getTime() >= 0)) {
                throughputStat.add(throughputStatDOToModel(throughputStatDOs.get(j)));
                k = j - 1;
            } else // 如果不满足if条件，则后面的数据也不用再遍历
            {
                break;
            }
        }
        if (throughputStat.size() > 0) {
            throughputInfo.setItems(throughputStat);
            throughputInfo.setSeconds(1 * 60L);
            throughputInfos.put(i, throughputInfo);
        }
    }
    return throughputInfos;
}