Examples with OperatorStats com.datatorrent.api.Stats.OperatorStats used on opensource projects

Example 6 with OperatorStats

use of com.datatorrent.api.Stats.OperatorStats in project apex-core by apache.

the class StreamingContainerManagerTest method testProcessHeartbeat.

@Test
public void testProcessHeartbeat() throws Exception {
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    Assert.assertEquals("number required containers", 1, plan.getContainers().size());
    PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    // assign container
    String containerId = "container1";
    StreamingContainerAgent sca = scm.assignContainer(new ContainerResource(0, containerId, "localhost", 512, 0, null), InetSocketAddress.createUnresolved("localhost", 0));
    Assert.assertNotNull(sca);
    Assert.assertEquals(PTContainer.State.ALLOCATED, o1p1.getContainer().getState());
    Assert.assertEquals(PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    ContainerStats cstats = new ContainerStats(containerId);
    ContainerHeartbeat hb = new ContainerHeartbeat();
    hb.setContainerStats(cstats);
    // get deploy request
    ContainerHeartbeatResponse chr = scm.processHeartbeat(hb);
    Assert.assertNotNull(chr.deployRequest);
    Assert.assertEquals("" + chr.deployRequest, 1, chr.deployRequest.size());
    Assert.assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
    Assert.assertEquals("state " + o1p1, PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    // first operator heartbeat
    OperatorHeartbeat ohb = new OperatorHeartbeat();
    ohb.setNodeId(o1p1.getId());
    ohb.setState(OperatorHeartbeat.DeployState.ACTIVE);
    OperatorStats stats = new OperatorStats();
    stats.checkpoint = new Checkpoint(2, 0, 0);
    stats.windowId = 3;
    stats.outputPorts = Lists.newArrayList();
    PortStats ps = new PortStats(TestGeneratorInputOperator.OUTPUT_PORT);
    ps.bufferServerBytes = 101;
    ps.tupleCount = 1;
    stats.outputPorts.add(ps);
    ohb.windowStats = Lists.newArrayList(stats);
    cstats.operators.add(ohb);
    // activate operator
    scm.processHeartbeat(hb);
    Assert.assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
    Assert.assertEquals("state " + o1p1, PTOperator.State.ACTIVE, o1p1.getState());
    Assert.assertEquals("tuples " + o1p1, 1, o1p1.stats.totalTuplesEmitted.get());
    Assert.assertEquals("tuples " + o1p1, 0, o1p1.stats.totalTuplesProcessed.get());
    Assert.assertEquals("window " + o1p1, 3, o1p1.stats.currentWindowId.get());
    Assert.assertEquals("port stats", 1, o1p1.stats.outputPortStatusList.size());
    PortStatus o1p1ps = o1p1.stats.outputPortStatusList.get(TestGeneratorInputOperator.OUTPUT_PORT);
    Assert.assertNotNull("port stats", o1p1ps);
    Assert.assertEquals("port stats", 1, o1p1ps.totalTuples);
    // second operator heartbeat
    stats = new OperatorStats();
    stats.checkpoint = new Checkpoint(2, 0, 0);
    stats.windowId = 4;
    stats.outputPorts = Lists.newArrayList();
    ps = new PortStats(TestGeneratorInputOperator.OUTPUT_PORT);
    ps.bufferServerBytes = 1;
    ps.tupleCount = 1;
    stats.outputPorts.add(ps);
    ohb.windowStats = Lists.newArrayList(stats);
    cstats.operators.clear();
    cstats.operators.add(ohb);
    scm.processHeartbeat(hb);
    Assert.assertEquals("tuples " + o1p1, 2, o1p1.stats.totalTuplesEmitted.get());
    Assert.assertEquals("window " + o1p1, 4, o1p1.stats.currentWindowId.get());
    Assert.assertEquals("statsQueue " + o1p1, 2, o1p1.stats.listenerStats.size());
    scm.processEvents();
    Assert.assertEquals("statsQueue " + o1p1, 0, o1p1.stats.listenerStats.size());
    Assert.assertEquals("lastStats " + o1p1, 2, o1p1.stats.lastWindowedStats.size());
}

Example 7 with OperatorStats

use of com.datatorrent.api.Stats.OperatorStats in project apex-core by apache.

the class StreamingContainerManager method processEvents.

public int processEvents() {
    for (PTOperator o : reportStats.keySet()) {
        List<OperatorStats> stats = o.stats.listenerStats.poll();
        if (stats != null) {
            // append into single list
            List<OperatorStats> moreStats;
            while ((moreStats = o.stats.listenerStats.poll()) != null) {
                stats.addAll(moreStats);
            }
        }
        o.stats.lastWindowedStats = stats;
        o.stats.operatorResponses = null;
        if (!o.stats.responses.isEmpty()) {
            o.stats.operatorResponses = new ArrayList<>();
            StatsListener.OperatorResponse operatorResponse;
            while ((operatorResponse = o.stats.responses.poll()) != null) {
                o.stats.operatorResponses.add(operatorResponse);
            }
        }
        if (o.stats.lastWindowedStats != null) {
            // call listeners only with non empty window list
            if (o.statsListeners != null) {
                plan.onStatusUpdate(o);
            }
        }
        reportStats.remove(o);
    }
    if (!this.shutdownOperators.isEmpty()) {
        synchronized (this.shutdownOperators) {
            Iterator<Map.Entry<Long, Set<PTOperator>>> it = shutdownOperators.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry<Long, Set<PTOperator>> windowAndOpers = it.next();
                if (windowAndOpers.getKey().longValue() <= this.committedWindowId || checkDownStreamOperators(windowAndOpers)) {
                    LOG.info("Removing inactive operators at window {} {}", Codec.getStringWindowId(windowAndOpers.getKey()), windowAndOpers.getValue());
                    for (PTOperator oper : windowAndOpers.getValue()) {
                        plan.removeTerminatedPartition(oper);
                    }
                    it.remove();
                }
            }
        }
    }
    if (!eventQueue.isEmpty()) {
        for (PTOperator oper : plan.getAllOperators().values()) {
            if (oper.getState() != PTOperator.State.ACTIVE) {
                LOG.debug("Skipping plan updates due to inactive operator {} {}", oper, oper.getState());
                return 0;
            }
        }
    }
    int count = 0;
    Runnable command;
    while ((command = this.eventQueue.poll()) != null) {
        eventQueueProcessing.set(true);
        try {
            command.run();
            count++;
        } catch (Exception e) {
            // TODO: handle error
            LOG.error("Failed to execute {}", command, e);
        }
        eventQueueProcessing.set(false);
    }
    if (count > 0) {
        try {
            checkpoint();
        } catch (Exception e) {
            throw new RuntimeException("Failed to checkpoint state.", e);
        }
    }
    return count;
}