Examples with TableStat com.alibaba.otter.shared.common.model.statistics.table.TableStat used on opensource projects

Example 1 with TableStat

use of com.alibaba.otter.shared.common.model.statistics.table.TableStat in project otter by alibaba.

the class StatsRemoteServiceImpl method onTableStat.

public void onTableStat(TableStatEvent event) {
    Assert.notNull(event);
    Assert.notNull(event.getStats());
    for (TableStat stat : event.getStats()) {
        tableStatService.updateTableStat(stat);
    }
}

Example 2 with TableStat

use of com.alibaba.otter.shared.common.model.statistics.table.TableStat in project otter by alibaba.

the class TableStatServiceImpl method tableHistoryStatDOToModel.

/**
 * 用于DO对象转化为Model对象
 *
 * @param TableHistoryStatDO
 * @return TableStat
 */
private TableStat tableHistoryStatDOToModel(TableHistoryStatDO tableHistoryStatDO) {
    TableStat tableStat = new TableStat();
    tableStat.setId(tableHistoryStatDO.getId());
    tableStat.setPipelineId(tableHistoryStatDO.getPipelineId());
    tableStat.setDataMediaPairId(tableHistoryStatDO.getDataMediaPairId());
    tableStat.setStartTime(tableHistoryStatDO.getStartTime());
    tableStat.setEndTime(tableHistoryStatDO.getEndTime());
    tableStat.setFileSize(tableHistoryStatDO.getFileSize());
    tableStat.setFileCount(tableHistoryStatDO.getFileCount());
    tableStat.setDeleteCount(tableHistoryStatDO.getDeleteCount());
    tableStat.setInsertCount(tableHistoryStatDO.getInsertCount());
    tableStat.setUpdateCount(tableHistoryStatDO.getUpdateCount());
    tableStat.setGmtCreate(tableHistoryStatDO.getGmtCreate());
    tableStat.setGmtModified(tableHistoryStatDO.getGmtModified());
    return tableStat;
}

Example 3 with TableStat

use of com.alibaba.otter.shared.common.model.statistics.table.TableStat in project otter by alibaba.

the class TableStatServiceImpl method updateTableStat.

/**
 * 先通过pipeLineId和DataMediaPairId在数据库里查找对应的tableStat，如果有，则增量更新对应的Table统计状态， 如果没有则将该数据插入
 */
public void updateTableStat(TableStat stat) {
    Assert.assertNotNull(stat);
    int affect = tableStatDao.modifyTableStat(tableStatModelToDo(stat));
    if (affect == 0) {
        tableStatDao.insertTableStat(tableStatModelToDo(stat));
    }
    if (stat.getStartTime() != null && stat.getEndTime() != null) {
        if (statUnit <= 0) {
            insertBehaviorHistory(stat);
        } else {
            synchronized (tableStats) {
                // 插入历史数据表
                TableStat old = tableStats.get(stat.getDataMediaPairId());
                if (old != null) {
                    // 合并数据
                    old.setInsertCount(stat.getInsertCount() + old.getInsertCount());
                    old.setUpdateCount(stat.getUpdateCount() + old.getUpdateCount());
                    old.setDeleteCount(stat.getDeleteCount() + old.getDeleteCount());
                    old.setFileCount(stat.getFileCount() + old.getFileCount());
                    old.setFileSize(stat.getFileSize() + old.getFileSize());
                    if (stat.getEndTime().after(old.getEndTime())) {
                        old.setEndTime(stat.getEndTime());
                    }
                    if (stat.getStartTime().before(old.getStartTime())) {
                        old.setStartTime(stat.getStartTime());
                    }
                } else {
                    tableStats.put(stat.getDataMediaPairId(), stat);
                }
            }
        }
    }
}

Example 4 with TableStat

use of com.alibaba.otter.shared.common.model.statistics.table.TableStat in project otter by alibaba.

the class OtterLoaderFactory method sendStat.

private void sendStat(Identity identity) {
    LoadThroughput throughput = loadStatsTracker.getStat(identity);
    Collection<LoadCounter> counters = throughput.getStats();
    Date endTime = new Date();
    // 处理table stat
    long fileSize = 0L;
    long fileCount = 0L;
    long rowSize = 0L;
    long rowCount = 0L;
    long mqSize = 0L;
    long mqCount = 0L;
    List<TableStat> tableStats = new ArrayList<TableStat>();
    for (LoadCounter counter : counters) {
        TableStat stat = new TableStat();
        stat.setPipelineId(identity.getPipelineId());
        stat.setDataMediaPairId(counter.getPairId());
        stat.setFileCount(counter.getFileCount().longValue());
        stat.setFileSize(counter.getFileSize().longValue());
        stat.setInsertCount(counter.getInsertCount().longValue());
        stat.setUpdateCount(counter.getUpdateCount().longValue());
        stat.setDeleteCount(counter.getDeleteCount().longValue());
        stat.setStartTime(new Date(throughput.getStartTime()));
        stat.setEndTime(endTime);
        // 5项中有一项不为空才通知
        if (!(stat.getFileCount().equals(0L) && stat.getFileSize().equals(0L) && stat.getInsertCount().equals(0L) && stat.getDeleteCount().equals(0L) && stat.getUpdateCount().equals(0L))) {
            tableStats.add(stat);
        }
        fileSize += counter.getFileSize().longValue();
        fileCount += counter.getFileCount().longValue();
        rowSize += counter.getRowSize().longValue();
        rowCount += counter.getRowCount().longValue();
        mqSize += counter.getMqSize().longValue();
        mqCount += counter.getMqCount().longValue();
    }
    if (!CollectionUtils.isEmpty(tableStats)) {
        statisticsClientService.sendTableStats(tableStats);
    }
    List<ThroughputStat> throughputStats = new ArrayList<ThroughputStat>();
    if (!(rowCount == 0 && rowSize == 0)) {
        // 处理Throughput stat
        ThroughputStat rowThroughputStat = new ThroughputStat();
        rowThroughputStat.setType(ThroughputType.ROW);
        rowThroughputStat.setPipelineId(identity.getPipelineId());
        rowThroughputStat.setNumber(rowCount);
        rowThroughputStat.setSize(rowSize);
        rowThroughputStat.setStartTime(new Date(throughput.getStartTime()));
        rowThroughputStat.setEndTime(endTime);
        throughputStats.add(rowThroughputStat);
    }
    if (!(fileCount == 0 && fileSize == 0)) {
        ThroughputStat fileThroughputStat = new ThroughputStat();
        fileThroughputStat.setType(ThroughputType.FILE);
        fileThroughputStat.setPipelineId(identity.getPipelineId());
        fileThroughputStat.setNumber(fileCount);
        fileThroughputStat.setSize(fileSize);
        fileThroughputStat.setStartTime(new Date(throughput.getStartTime()));
        fileThroughputStat.setEndTime(endTime);
        throughputStats.add(fileThroughputStat);
    }
    // add by 2012-07-06 for mq loader
    if (!(mqCount == 0 && mqSize == 0)) {
        ThroughputStat mqThroughputStat = new ThroughputStat();
        mqThroughputStat.setType(ThroughputType.MQ);
        mqThroughputStat.setPipelineId(identity.getPipelineId());
        mqThroughputStat.setNumber(mqCount);
        mqThroughputStat.setSize(mqSize);
        mqThroughputStat.setStartTime(new Date(throughput.getStartTime()));
        mqThroughputStat.setEndTime(endTime);
        throughputStats.add(mqThroughputStat);
    }
    if (!CollectionUtils.isEmpty(throughputStats)) {
        statisticsClientService.sendThroughputs(throughputStats);
    }
}

Example 5 with TableStat

use of com.alibaba.otter.shared.common.model.statistics.table.TableStat in project otter by alibaba.

the class DataMediaPairList method execute.

public void execute(@Param("pipelineId") Long pipelineId, Context context) throws Exception {
    // Pipeline pipeline = pipelineService.findById(pipelineId);
    Channel channel = channelService.findByPipelineId(pipelineId);
    List<DataMediaPair> dataMediaPairs = dataMediaPairService.listByPipelineId(pipelineId);
    Map<Long, TableStat> tableStatMap = new HashMap<Long, TableStat>(dataMediaPairs.size(), 1f);
    List<TableStat> tableStats = tableStatService.listTableStat(pipelineId);
    for (DataMediaPair dataMediaPair : dataMediaPairs) {
        int flag = 0;
        for (TableStat tableStat : tableStats) {
            if (dataMediaPair.getId().equals(tableStat.getDataMediaPairId())) {
                tableStatMap.put(dataMediaPair.getId(), tableStat);
                flag = 1;
                break;
            }
        }
        if (flag == 0) {
            TableStat tableStat = new TableStat();
            tableStat.setFileSize(0L);
            tableStat.setFileCount(0L);
            tableStat.setDeleteCount(0L);
            tableStat.setUpdateCount(0L);
            tableStat.setInsertCount(0L);
            // tableStat.setGmtModified(dataMediaPair.getGmtModified());
            tableStatMap.put(dataMediaPair.getId(), tableStat);
        }
    }
    context.put("dataMediaPairs", dataMediaPairs);
    // 通过PipelineId不能获取到Channel状态，所以需要传递Channel对象
    context.put("channel", channel);
    context.put("pipelineId", pipelineId);
    context.put("tableStatMap", tableStatMap);
}