Examples with LoadCounter com.alibaba.otter.node.etl.load.loader.LoadStatsTracker.LoadCounter used on opensource projects

Example 1 with LoadCounter

use of com.alibaba.otter.node.etl.load.loader.LoadStatsTracker.LoadCounter in project otter by alibaba.

the class OtterLoaderFactory method sendStat.

private void sendStat(Identity identity) {
    LoadThroughput throughput = loadStatsTracker.getStat(identity);
    Collection<LoadCounter> counters = throughput.getStats();
    Date endTime = new Date();
    // 处理table stat
    long fileSize = 0L;
    long fileCount = 0L;
    long rowSize = 0L;
    long rowCount = 0L;
    long mqSize = 0L;
    long mqCount = 0L;
    List<TableStat> tableStats = new ArrayList<TableStat>();
    for (LoadCounter counter : counters) {
        TableStat stat = new TableStat();
        stat.setPipelineId(identity.getPipelineId());
        stat.setDataMediaPairId(counter.getPairId());
        stat.setFileCount(counter.getFileCount().longValue());
        stat.setFileSize(counter.getFileSize().longValue());
        stat.setInsertCount(counter.getInsertCount().longValue());
        stat.setUpdateCount(counter.getUpdateCount().longValue());
        stat.setDeleteCount(counter.getDeleteCount().longValue());
        stat.setStartTime(new Date(throughput.getStartTime()));
        stat.setEndTime(endTime);
        // 5项中有一项不为空才通知
        if (!(stat.getFileCount().equals(0L) && stat.getFileSize().equals(0L) && stat.getInsertCount().equals(0L) && stat.getDeleteCount().equals(0L) && stat.getUpdateCount().equals(0L))) {
            tableStats.add(stat);
        }
        fileSize += counter.getFileSize().longValue();
        fileCount += counter.getFileCount().longValue();
        rowSize += counter.getRowSize().longValue();
        rowCount += counter.getRowCount().longValue();
        mqSize += counter.getMqSize().longValue();
        mqCount += counter.getMqCount().longValue();
    }
    if (!CollectionUtils.isEmpty(tableStats)) {
        statisticsClientService.sendTableStats(tableStats);
    }
    List<ThroughputStat> throughputStats = new ArrayList<ThroughputStat>();
    if (!(rowCount == 0 && rowSize == 0)) {
        // 处理Throughput stat
        ThroughputStat rowThroughputStat = new ThroughputStat();
        rowThroughputStat.setType(ThroughputType.ROW);
        rowThroughputStat.setPipelineId(identity.getPipelineId());
        rowThroughputStat.setNumber(rowCount);
        rowThroughputStat.setSize(rowSize);
        rowThroughputStat.setStartTime(new Date(throughput.getStartTime()));
        rowThroughputStat.setEndTime(endTime);
        throughputStats.add(rowThroughputStat);
    }
    if (!(fileCount == 0 && fileSize == 0)) {
        ThroughputStat fileThroughputStat = new ThroughputStat();
        fileThroughputStat.setType(ThroughputType.FILE);
        fileThroughputStat.setPipelineId(identity.getPipelineId());
        fileThroughputStat.setNumber(fileCount);
        fileThroughputStat.setSize(fileSize);
        fileThroughputStat.setStartTime(new Date(throughput.getStartTime()));
        fileThroughputStat.setEndTime(endTime);
        throughputStats.add(fileThroughputStat);
    }
    // add by 2012-07-06 for mq loader
    if (!(mqCount == 0 && mqSize == 0)) {
        ThroughputStat mqThroughputStat = new ThroughputStat();
        mqThroughputStat.setType(ThroughputType.MQ);
        mqThroughputStat.setPipelineId(identity.getPipelineId());
        mqThroughputStat.setNumber(mqCount);
        mqThroughputStat.setSize(mqSize);
        mqThroughputStat.setStartTime(new Date(throughput.getStartTime()));
        mqThroughputStat.setEndTime(endTime);
        throughputStats.add(mqThroughputStat);
    }
    if (!CollectionUtils.isEmpty(throughputStats)) {
        statisticsClientService.sendThroughputs(throughputStats);
    }
}

Example 2 with LoadCounter

use of com.alibaba.otter.node.etl.load.loader.LoadStatsTracker.LoadCounter in project otter by alibaba.

the class DbLoadAction method processStat.

private void processStat(EventData data, DbLoadContext context) {
    LoadThroughput throughput = loadStatsTracker.getStat(context.getIdentity());
    LoadCounter counter = throughput.getStat(data.getPairId());
    EventType type = data.getEventType();
    if (type.isInsert()) {
        counter.getInsertCount().incrementAndGet();
    } else if (type.isUpdate()) {
        counter.getUpdateCount().incrementAndGet();
    } else if (type.isDelete()) {
        counter.getDeleteCount().incrementAndGet();
    }
    counter.getRowCount().incrementAndGet();
    counter.getRowSize().addAndGet(calculateSize(data));
}

Example 3 with LoadCounter

use of com.alibaba.otter.node.etl.load.loader.LoadStatsTracker.LoadCounter in project otter by alibaba.

the class FileLoadAction method dryRun.

private void dryRun(FileLoadContext context, List<FileData> fileDatas, File rootDir) {
    for (FileData fileData : fileDatas) {
        boolean isLocal = StringUtils.isBlank(fileData.getNameSpace());
        String entryName = null;
        if (true == isLocal) {
            entryName = FilenameUtils.getPath(fileData.getPath()) + FilenameUtils.getName(fileData.getPath());
        } else {
            entryName = fileData.getNameSpace() + File.separator + fileData.getPath();
        }
        File sourceFile = new File(rootDir, entryName);
        if (true == sourceFile.exists() && false == sourceFile.isDirectory()) {
            if (false == isLocal) {
                throw new LoadException(fileData + " is not support!");
            } else {
                // 记录一下文件的meta信息
                fileData.setSize(sourceFile.length());
                fileData.setLastModifiedTime(sourceFile.lastModified());
                context.getProcessedDatas().add(fileData);
            }
            LoadCounter counter = loadStatsTracker.getStat(context.getIdentity()).getStat(fileData.getPairId());
            counter.getFileCount().incrementAndGet();
            counter.getFileSize().addAndGet(fileData.getSize());
        } else if (fileData.getEventType().isDelete()) {
            // 删除对应的文件
            if (false == isLocal) {
                throw new LoadException(fileData + " is not support!");
            } else {
                context.getProcessedDatas().add(fileData);
            }
        } else {
            // 失败记录
            context.getFailedDatas().add(fileData);
        }
    }
}

Example 4 with LoadCounter

use of com.alibaba.otter.node.etl.load.loader.LoadStatsTracker.LoadCounter in project otter by alibaba.

the class FileLoadAction method doMove.

private void doMove(FileLoadContext context, File rootDir, FileData fileData) throws IOException {
    boolean isLocal = StringUtils.isBlank(fileData.getNameSpace());
    String entryName = null;
    if (true == isLocal) {
        entryName = FilenameUtils.getPath(fileData.getPath()) + FilenameUtils.getName(fileData.getPath());
    } else {
        entryName = fileData.getNameSpace() + File.separator + fileData.getPath();
    }
    File sourceFile = new File(rootDir, entryName);
    if (true == sourceFile.exists() && false == sourceFile.isDirectory()) {
        if (false == isLocal) {
            throw new LoadException(fileData + " is not support!");
        } else {
            File targetFile = new File(fileData.getPath());
            // copy to product path
            NioUtils.copy(sourceFile, targetFile, retry);
            if (true == targetFile.exists()) {
                // 记录一下文件的meta信息
                fileData.setSize(sourceFile.length());
                fileData.setLastModifiedTime(sourceFile.lastModified());
                context.getProcessedDatas().add(fileData);
            } else {
                throw new LoadException(String.format("copy/rename [%s] to [%s] failed by unknow reason", sourceFile.getPath(), targetFile.getPath()));
            }
        }
        LoadCounter counter = loadStatsTracker.getStat(context.getIdentity()).getStat(fileData.getPairId());
        counter.getFileCount().incrementAndGet();
        counter.getFileSize().addAndGet(fileData.getSize());
    } else if (fileData.getEventType().isDelete()) {
        // 删除对应的文件
        if (false == isLocal) {
            throw new LoadException(fileData + " is not support!");
        } else {
            File targetFile = new File(fileData.getPath());
            if (NioUtils.delete(targetFile, retry)) {
                context.getProcessedDatas().add(fileData);
            } else {
                context.getFailedDatas().add(fileData);
            }
        }
    } else {
        // 失败记录
        context.getFailedDatas().add(fileData);
    }
}