Examples with PortStats com.datatorrent.api.Stats.OperatorStats.PortStats used on opensource projects

Example 1 with PortStats

use of com.datatorrent.api.Stats.OperatorStats.PortStats in project apex-core by apache.

the class TupleRecorderCollection method collected.

@Handler
public void collected(ContainerStatsEvent cse) {
    ContainerStats stats = cse.getContainerStats();
    for (OperatorHeartbeat node : stats.operators) {
        for (OperatorStats os : node.windowStats) {
            if (os.inputPorts != null) {
                for (PortStats ps : os.inputPorts) {
                    ps.recordingId = null;
                }
            }
            if (os.outputPorts != null) {
                for (PortStats ps : os.outputPorts) {
                    ps.recordingId = null;
                }
            }
        }
    }
    for (OperatorHeartbeat node : stats.operators) {
        String recordingId;
        TupleRecorder tupleRecorder = get(new OperatorIdPortNamePair(node.nodeId, null));
        if (tupleRecorder == null) {
            recordingId = null;
            for (Map.Entry<OperatorIdPortNamePair, TupleRecorder> entry : this.entrySet()) {
                if (entry.getKey().operatorId == node.nodeId) {
                    for (OperatorStats os : node.windowStats) {
                        if (os.inputPorts != null) {
                            for (PortStats ps : os.inputPorts) {
                                if (ps.id.equals(entry.getKey().portName)) {
                                    ps.recordingId = entry.getValue().getId();
                                    break;
                                }
                            }
                        }
                        if (os.outputPorts != null) {
                            for (PortStats ps : os.outputPorts) {
                                if (ps.id.equals(entry.getKey().portName)) {
                                    ps.recordingId = entry.getValue().getId();
                                    break;
                                }
                            }
                        }
                    }
                }
            }
        } else {
            recordingId = tupleRecorder.getId();
        }
        for (OperatorStats os : node.windowStats) {
            os.recordingId = recordingId;
        }
    }
}

Example 2 with PortStats

use of com.datatorrent.api.Stats.OperatorStats.PortStats in project apex-core by apache.

the class StatsTest method testPortStatsPropagation.

/**
 * Verify buffer server bytes and tuple count.
 *
 * @throws Exception
 */
@Test
@SuppressWarnings("SleepWhileInLoop")
public void testPortStatsPropagation() throws Exception {
    int tupleCount = 10;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target").getAbsolutePath();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
    TestInputStatsListener testInputStatsListener = new TestInputStatsListener();
    dag.setOperatorAttribute(testOper, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { testInputStatsListener }));
    testOper.setMaxTuples(tupleCount);
    testOper.setEmitInterval(0);
    TestCollector collector = dag.addOperator("Collector", new TestCollector());
    TestCollectorStatsListener testCollectorStatsListener = new TestCollectorStatsListener();
    dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { testCollectorStatsListener }));
    dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(null);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    Assert.assertFalse("input operator stats", testInputStatsListener.inputOperatorStats.isEmpty());
    Assert.assertFalse("collector operator stats", testCollectorStatsListener.collectorOperatorStats.isEmpty());
    try {
        int outputPortTupleCount = 0;
        long outputPortBufferServerBytes = 0L;
        for (Iterator<OperatorStats> it = testInputStatsListener.inputOperatorStats.iterator(); it.hasNext(); ) {
            OperatorStats operatorStats = it.next();
            for (PortStats outputPortStats : operatorStats.outputPorts) {
                outputPortTupleCount += outputPortStats.tupleCount;
                outputPortBufferServerBytes += outputPortStats.bufferServerBytes;
            }
        }
        int inputPortTupleCount = 0;
        long inputPortBufferServerBytes = 0L;
        for (Iterator<OperatorStats> it = testCollectorStatsListener.collectorOperatorStats.iterator(); it.hasNext(); ) {
            OperatorStats operatorStats = it.next();
            for (PortStats inputPortStats : operatorStats.inputPorts) {
                inputPortTupleCount += inputPortStats.tupleCount;
                inputPortBufferServerBytes += inputPortStats.bufferServerBytes;
            }
        }
        Assert.assertEquals("Tuple Count emitted", tupleCount, outputPortTupleCount);
        Assert.assertTrue("Buffer server bytes", inputPortBufferServerBytes > 0);
        Assert.assertEquals("Tuple Count processed", tupleCount, inputPortTupleCount);
        Assert.assertTrue("Buffer server bytes", outputPortBufferServerBytes > 0);
    } finally {
        lc.shutdown();
    }
}

Example 3 with PortStats

use of com.datatorrent.api.Stats.OperatorStats.PortStats in project apex-core by apache.

the class StreamingContainerManagerTest method testProcessHeartbeat.

@Test
public void testProcessHeartbeat() throws Exception {
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    Assert.assertEquals("number required containers", 1, plan.getContainers().size());
    PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    // assign container
    String containerId = "container1";
    StreamingContainerAgent sca = scm.assignContainer(new ContainerResource(0, containerId, "localhost", 512, 0, null), InetSocketAddress.createUnresolved("localhost", 0));
    Assert.assertNotNull(sca);
    Assert.assertEquals(PTContainer.State.ALLOCATED, o1p1.getContainer().getState());
    Assert.assertEquals(PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    ContainerStats cstats = new ContainerStats(containerId);
    ContainerHeartbeat hb = new ContainerHeartbeat();
    hb.setContainerStats(cstats);
    // get deploy request
    ContainerHeartbeatResponse chr = scm.processHeartbeat(hb);
    Assert.assertNotNull(chr.deployRequest);
    Assert.assertEquals("" + chr.deployRequest, 1, chr.deployRequest.size());
    Assert.assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
    Assert.assertEquals("state " + o1p1, PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    // first operator heartbeat
    OperatorHeartbeat ohb = new OperatorHeartbeat();
    ohb.setNodeId(o1p1.getId());
    ohb.setState(OperatorHeartbeat.DeployState.ACTIVE);
    OperatorStats stats = new OperatorStats();
    stats.checkpoint = new Checkpoint(2, 0, 0);
    stats.windowId = 3;
    stats.outputPorts = Lists.newArrayList();
    PortStats ps = new PortStats(TestGeneratorInputOperator.OUTPUT_PORT);
    ps.bufferServerBytes = 101;
    ps.tupleCount = 1;
    stats.outputPorts.add(ps);
    ohb.windowStats = Lists.newArrayList(stats);
    cstats.operators.add(ohb);
    // activate operator
    scm.processHeartbeat(hb);
    Assert.assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
    Assert.assertEquals("state " + o1p1, PTOperator.State.ACTIVE, o1p1.getState());
    Assert.assertEquals("tuples " + o1p1, 1, o1p1.stats.totalTuplesEmitted.get());
    Assert.assertEquals("tuples " + o1p1, 0, o1p1.stats.totalTuplesProcessed.get());
    Assert.assertEquals("window " + o1p1, 3, o1p1.stats.currentWindowId.get());
    Assert.assertEquals("port stats", 1, o1p1.stats.outputPortStatusList.size());
    PortStatus o1p1ps = o1p1.stats.outputPortStatusList.get(TestGeneratorInputOperator.OUTPUT_PORT);
    Assert.assertNotNull("port stats", o1p1ps);
    Assert.assertEquals("port stats", 1, o1p1ps.totalTuples);
    // second operator heartbeat
    stats = new OperatorStats();
    stats.checkpoint = new Checkpoint(2, 0, 0);
    stats.windowId = 4;
    stats.outputPorts = Lists.newArrayList();
    ps = new PortStats(TestGeneratorInputOperator.OUTPUT_PORT);
    ps.bufferServerBytes = 1;
    ps.tupleCount = 1;
    stats.outputPorts.add(ps);
    ohb.windowStats = Lists.newArrayList(stats);
    cstats.operators.clear();
    cstats.operators.add(ohb);
    scm.processHeartbeat(hb);
    Assert.assertEquals("tuples " + o1p1, 2, o1p1.stats.totalTuplesEmitted.get());
    Assert.assertEquals("window " + o1p1, 4, o1p1.stats.currentWindowId.get());
    Assert.assertEquals("statsQueue " + o1p1, 2, o1p1.stats.listenerStats.size());
    scm.processEvents();
    Assert.assertEquals("statsQueue " + o1p1, 0, o1p1.stats.listenerStats.size());
    Assert.assertEquals("lastStats " + o1p1, 2, o1p1.stats.lastWindowedStats.size());
}