Examples with LogicalOperatorInfo com.datatorrent.stram.webapp.LogicalOperatorInfo used on opensource projects

Example 1 with LogicalOperatorInfo

use of com.datatorrent.stram.webapp.LogicalOperatorInfo in project apex-core by apache.

the class AppDataPushAgent method getPushData.

private JSONObject getPushData() {
    // assemble the json that contains the app stats and logical operator stats and counters
    JSONObject json = new JSONObject();
    try {
        json.put("type", DATA);
        json.put("appId", dnmgr.getLogicalPlan().getValue(DAGContext.APPLICATION_ID));
        json.put("appName", dnmgr.getLogicalPlan().getValue(DAGContext.APPLICATION_NAME));
        json.put("appUser", appContext.getUser());
        List<LogicalOperatorInfo> logicalOperatorInfoList = dnmgr.getLogicalOperatorInfoList();
        JSONObject logicalOperators = new JSONObject();
        for (LogicalOperatorInfo logicalOperator : logicalOperatorInfoList) {
            JSONObject logicalOperatorJson = extractFields(logicalOperator);
            JSONArray metricsList = new JSONArray();
            Queue<Pair<Long, Map<String, Object>>> windowMetrics = dnmgr.getWindowMetrics(logicalOperator.name);
            if (windowMetrics != null) {
                while (!windowMetrics.isEmpty()) {
                    Pair<Long, Map<String, Object>> metrics = windowMetrics.remove();
                    long windowId = metrics.first;
                    // metric name, aggregated value
                    Map<String, Object> aggregates = metrics.second;
                    long now = System.currentTimeMillis();
                    if (!operatorsSchemaLastSentTime.containsKey(logicalOperator.name) || (metricsTransport.getSchemaResendInterval() > 0 && operatorsSchemaLastSentTime.get(logicalOperator.name) < now - metricsTransport.getSchemaResendInterval())) {
                        try {
                            pushMetricsSchema(dnmgr.getLogicalPlan().getOperatorMeta(logicalOperator.name), aggregates);
                            operatorsSchemaLastSentTime.put(logicalOperator.name, now);
                        } catch (IOException ex) {
                            LOG.error("Cannot push metrics schema", ex);
                        }
                    }
                    JSONObject metricsItem = new JSONObject();
                    metricsItem.put("_windowId", windowId);
                    long windowToMillis = dnmgr.windowIdToMillis(windowId);
                    LOG.debug("metric window {} time {}", windowId, windowToMillis);
                    metricsItem.put("_time", windowToMillis);
                    for (Map.Entry<String, Object> entry : aggregates.entrySet()) {
                        String metricName = entry.getKey();
                        Object aggregateValue = entry.getValue();
                        metricsItem.put(metricName, aggregateValue);
                    }
                    metricsList.put(metricsItem);
                }
            }
            logicalOperatorJson.put("metrics", metricsList);
            logicalOperators.put(logicalOperator.name, logicalOperatorJson);
        }
        json.put("time", System.currentTimeMillis());
        json.put("logicalOperators", logicalOperators);
        json.put("stats", extractFields(appContext.getStats()));
    } catch (JSONException ex) {
        throw new RuntimeException(ex);
    }
    return json;
}

Example 2 with LogicalOperatorInfo

use of com.datatorrent.stram.webapp.LogicalOperatorInfo in project apex-core by apache.

the class StreamingContainerManager method getLogicalOperatorInfoList.

public List<LogicalOperatorInfo> getLogicalOperatorInfoList() {
    List<LogicalOperatorInfo> infoList = new ArrayList<>();
    Collection<OperatorMeta> allOperators = getLogicalPlan().getAllOperators();
    for (OperatorMeta operatorMeta : allOperators) {
        infoList.add(fillLogicalOperatorInfo(operatorMeta));
    }
    return infoList;
}

Example 3 with LogicalOperatorInfo

use of com.datatorrent.stram.webapp.LogicalOperatorInfo in project apex-core by apache.

the class StreamingContainerManagerTest method testLatency.

@Test
public void testLatency() throws Exception {
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    HighLatencyTestOperator o3 = dag.addOperator("o3", HighLatencyTestOperator.class);
    GenericTestOperator o4 = dag.addOperator("o4", GenericTestOperator.class);
    // 5 seconds
    long latency = 5000;
    o3.setLatency(latency);
    dag.addStream("o1.outport", o1.outport, o2.inport1, o3.inport1);
    dag.addStream("o2.outport1", o2.outport1, o4.inport1);
    dag.addStream("o3.outport1", o3.outport1, o4.inport2);
    // 1 second
    dag.setAttribute(Context.DAGContext.STATS_MAX_ALLOWABLE_WINDOWS_LAG, 2);
    StramLocalCluster lc = new StramLocalCluster(dag);
    StreamingContainerManager dnmgr = lc.dnmgr;
    lc.runAsync();
    Thread.sleep(10000);
    LogicalOperatorInfo o1Info = dnmgr.getLogicalOperatorInfo("o1");
    LogicalOperatorInfo o2Info = dnmgr.getLogicalOperatorInfo("o2");
    LogicalOperatorInfo o3Info = dnmgr.getLogicalOperatorInfo("o3");
    LogicalOperatorInfo o4Info = dnmgr.getLogicalOperatorInfo("o4");
    Assert.assertEquals("Input operator latency must be zero", 0, o1Info.latencyMA);
    Assert.assertTrue("Latency must be greater than or equal to zero", o2Info.latencyMA >= 0);
    Assert.assertTrue("Actual latency must be greater than the artificially introduced latency", o3Info.latencyMA > latency);
    Assert.assertTrue("Latency must be greater than or equal to zero", o4Info.latencyMA >= 0);
    StreamingContainerManager.CriticalPathInfo criticalPathInfo = dnmgr.getCriticalPathInfo();
    Assert.assertArrayEquals("Critical Path must be the path in the DAG that includes the HighLatencyTestOperator", new Integer[] { o1Info.partitions.iterator().next(), o3Info.partitions.iterator().next(), o4Info.partitions.iterator().next() }, criticalPathInfo.path.toArray());
    Assert.assertTrue("Whole DAG latency must be greater than the artificially introduced latency", criticalPathInfo.latency > latency);
    lc.shutdown();
}

Example 4 with LogicalOperatorInfo

use of com.datatorrent.stram.webapp.LogicalOperatorInfo in project apex-core by apache.

the class StreamingContainerManager method fillLogicalOperatorInfo.

private LogicalOperatorInfo fillLogicalOperatorInfo(OperatorMeta operator) {
    LogicalOperatorInfo loi = new LogicalOperatorInfo();
    loi.name = operator.getName();
    loi.className = operator.getOperator().getClass().getName();
    loi.totalTuplesEmitted = operator.getStatus().totalTuplesEmitted;
    loi.totalTuplesProcessed = operator.getStatus().totalTuplesProcessed;
    loi.failureCount = operator.getStatus().failureCount;
    loi.status = new HashMap<>();
    loi.partitions = new TreeSet<>();
    loi.unifiers = new TreeSet<>();
    loi.containerIds = new TreeSet<>();
    loi.hosts = new TreeSet<>();
    Collection<PTOperator> physicalOperators = getPhysicalPlan().getAllOperators(operator);
    NumberAggregate.LongAggregate checkpointTimeAggregate = new NumberAggregate.LongAggregate();
    for (PTOperator physicalOperator : physicalOperators) {
        OperatorStatus os = physicalOperator.stats;
        if (physicalOperator.isUnifier()) {
            loi.unifiers.add(physicalOperator.getId());
        } else {
            loi.partitions.add(physicalOperator.getId());
            // exclude unifier, not sure if we should include it in the future
            loi.tuplesEmittedPSMA += os.tuplesEmittedPSMA.get();
            loi.tuplesProcessedPSMA += os.tuplesProcessedPSMA.get();
            // calculate maximum latency for all partitions
            long latency = calculateLatency(physicalOperator);
            if (latency > loi.latencyMA) {
                loi.latencyMA = latency;
            }
            checkpointTimeAggregate.addNumber(os.checkpointTimeMA.getAvg());
        }
        loi.cpuPercentageMA += os.cpuNanosPMSMA.getAvg() / 10000;
        if (os.lastHeartbeat != null && (loi.lastHeartbeat == 0 || loi.lastHeartbeat > os.lastHeartbeat.getGeneratedTms())) {
            loi.lastHeartbeat = os.lastHeartbeat.getGeneratedTms();
        }
        long currentWindowId = toWsWindowId(os.currentWindowId.get());
        if (loi.currentWindowId == 0 || loi.currentWindowId > currentWindowId) {
            loi.currentWindowId = currentWindowId;
        }
        MutableInt count = loi.status.get(physicalOperator.getState().toString());
        if (count == null) {
            count = new MutableInt();
            loi.status.put(physicalOperator.getState().toString(), count);
        }
        count.increment();
        if (physicalOperator.getRecoveryCheckpoint() != null) {
            long recoveryWindowId = toWsWindowId(physicalOperator.getRecoveryCheckpoint().windowId);
            if (loi.recoveryWindowId == 0 || loi.recoveryWindowId > recoveryWindowId) {
                loi.recoveryWindowId = recoveryWindowId;
            }
        }
        PTContainer container = physicalOperator.getContainer();
        if (container != null) {
            String externalId = container.getExternalId();
            if (externalId != null) {
                loi.containerIds.add(externalId);
                loi.hosts.add(container.host);
            }
        }
    }
    if (physicalOperators.size() > 0 && checkpointTimeAggregate.getAvg() != null) {
        loi.checkpointTimeMA = checkpointTimeAggregate.getAvg().longValue();
        loi.counters = latestLogicalCounters.get(operator.getName());
        loi.autoMetrics = latestLogicalMetrics.get(operator.getName());
    }
    return loi;
}