Examples with OperatorInfo com.datatorrent.stram.webapp.OperatorInfo used on opensource projects

Example 1 with OperatorInfo

use of com.datatorrent.stram.webapp.OperatorInfo in project apex-core by apache.

the class StreamingContainerManager method fillPhysicalOperatorInfo.

private OperatorInfo fillPhysicalOperatorInfo(PTOperator operator) {
    OperatorInfo oi = new OperatorInfo();
    oi.container = operator.getContainer().getExternalId();
    oi.host = operator.getContainer().host;
    oi.id = Integer.toString(operator.getId());
    oi.name = operator.getName();
    oi.className = operator.getOperatorMeta().getOperator().getClass().getName();
    oi.status = operator.getState().toString();
    if (operator.isUnifier()) {
        oi.unifierClass = operator.getUnifierClass().getName();
    }
    oi.logicalName = operator.getOperatorMeta().getName();
    OperatorStatus os = operator.stats;
    oi.recordingId = os.recordingId;
    oi.totalTuplesProcessed = os.totalTuplesProcessed.get();
    oi.totalTuplesEmitted = os.totalTuplesEmitted.get();
    oi.tuplesProcessedPSMA = os.tuplesProcessedPSMA.get();
    oi.tuplesEmittedPSMA = os.tuplesEmittedPSMA.get();
    oi.cpuPercentageMA = os.cpuNanosPMSMA.getAvg() / 10000;
    oi.latencyMA = os.latencyMA.getAvg();
    oi.failureCount = operator.failureCount;
    oi.recoveryWindowId = toWsWindowId(operator.getRecoveryCheckpoint().windowId);
    oi.currentWindowId = toWsWindowId(os.currentWindowId.get());
    if (os.lastHeartbeat != null) {
        oi.lastHeartbeat = os.lastHeartbeat.getGeneratedTms();
    }
    if (os.checkpointStats != null) {
        oi.checkpointTime = os.checkpointStats.checkpointTime;
        oi.checkpointStartTime = os.checkpointStats.checkpointStartTime;
    }
    oi.checkpointTimeMA = os.checkpointTimeMA.getAvg();
    for (PortStatus ps : os.inputPortStatusList.values()) {
        PortInfo pinfo = new PortInfo();
        pinfo.name = ps.portName;
        pinfo.type = "input";
        pinfo.totalTuples = ps.totalTuples;
        pinfo.tuplesPSMA = Math.round(ps.tuplesPMSMA.getAvg() * 1000);
        pinfo.bufferServerBytesPSMA = Math.round(ps.bufferServerBytesPMSMA.getAvg() * 1000);
        pinfo.queueSizeMA = ps.queueSizeMA.getAvg();
        pinfo.recordingId = ps.recordingId;
        oi.addPort(pinfo);
    }
    for (PortStatus ps : os.outputPortStatusList.values()) {
        PortInfo pinfo = new PortInfo();
        pinfo.name = ps.portName;
        pinfo.type = "output";
        pinfo.totalTuples = ps.totalTuples;
        pinfo.tuplesPSMA = Math.round(ps.tuplesPMSMA.getAvg() * 1000);
        pinfo.bufferServerBytesPSMA = Math.round(ps.bufferServerBytesPMSMA.getAvg() * 1000);
        pinfo.recordingId = ps.recordingId;
        oi.addPort(pinfo);
    }
    oi.counters = os.getLastWindowedStats().size() > 0 ? os.getLastWindowedStats().get(os.getLastWindowedStats().size() - 1).counters : null;
    oi.metrics = os.getLastWindowedStats().size() > 0 ? os.getLastWindowedStats().get(os.getLastWindowedStats().size() - 1).metrics : null;
    return oi;
}

Example 2 with OperatorInfo

use of com.datatorrent.stram.webapp.OperatorInfo in project apex-core by apache.

the class FSStatsRecorder method recordOperators.

@Override
public void recordOperators(List<OperatorInfo> operatorList, long timestamp) throws IOException {
    for (OperatorInfo operatorInfo : operatorList) {
        FSPartFileCollection operatorStorage;
        if (!logicalOperatorStorageMap.containsKey(operatorInfo.name)) {
            operatorStorage = new FSPartFileCollection();
            operatorStorage.setBasePath(basePath + "/operators/" + operatorInfo.name);
            operatorStorage.setup();
            operatorStorage.writeMetaData((VERSION + "\n").getBytes());
            logicalOperatorStorageMap.put(operatorInfo.name, operatorStorage);
        } else {
            operatorStorage = logicalOperatorStorageMap.get(operatorInfo.name);
        }
        if (!knownOperators.contains(operatorInfo.id)) {
            knownOperators.add(operatorInfo.id);
            Map<String, Object> fieldMap = extractRecordFields(operatorInfo, "meta");
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            Slice f = streamCodec.toByteArray(fieldMap);
            bos.write(f.buffer, f.offset, f.length);
            bos.write("\n".getBytes());
            queue.add(new WriteOperation(operatorStorage, bos.toByteArray(), true));
        }
        Map<String, Object> fieldMap = extractRecordFields(operatorInfo, "stats");
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        Slice f = streamCodec.toByteArray(fieldMap);
        bos.write((operatorInfo.id + ":").getBytes());
        bos.write((String.valueOf(timestamp) + ":").getBytes());
        bos.write(f.buffer, f.offset, f.length);
        bos.write("\n".getBytes());
        queue.add(new WriteOperation(operatorStorage, bos.toByteArray(), false));
    }
}