Examples with StatsListener com.datatorrent.api.StatsListener used on opensource projects

Example 16 with StatsListener

use of com.datatorrent.api.StatsListener in project apex-core by apache.

the class PartitioningTest method testDynamicDefaultPartitioning.

//@Ignore
@Test
@SuppressWarnings("SleepWhileInLoop")
public void testDynamicDefaultPartitioning() throws Exception {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 5);
    File checkpointDir = new File(TEST_OUTPUT_DIR, "testDynamicDefaultPartitioning");
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(checkpointDir.getPath(), null));
    CollectorOperator.receivedTuples.clear();
    TestInputOperator<Integer> input = dag.addOperator("input", new TestInputOperator<Integer>());
    input.blockEndStream = true;
    CollectorOperator collector = dag.addOperator("partitionedCollector", new CollectorOperator());
    collector.prefix = "" + System.identityHashCode(collector);
    dag.setOperatorAttribute(collector, OperatorContext.PARTITIONER, new StatelessPartitioner<CollectorOperator>(2));
    dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitionLoadWatch() }));
    dag.addStream("fromInput", input.output, collector.input);
    CollectorOperator singleCollector = dag.addOperator("singleCollector", new CollectorOperator());
    singleCollector.prefix = "" + System.identityHashCode(singleCollector);
    dag.addStream("toSingleCollector", collector.output, singleCollector.input);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    List<PTOperator> partitions = assertNumberPartitions(2, lc, dag.getMeta(collector));
    Set<PTContainer> containers = Sets.newHashSet();
    for (PTOperator oper : partitions) {
        containers.add(oper.getContainer());
    }
    Assert.assertTrue("Number of containers are 4", 4 == lc.dnmgr.getPhysicalPlan().getContainers().size());
    PTOperator splitPartition = partitions.get(0);
    PartitionLoadWatch.put(splitPartition, 1);
    LOG.debug("Triggered split for {}", splitPartition);
    int count = 0;
    long startMillis = System.currentTimeMillis();
    while (count == 0 && startMillis > System.currentTimeMillis() - StramTestSupport.DEFAULT_TIMEOUT_MILLIS) {
        // yield
        sleep(20);
        count += lc.dnmgr.processEvents();
    }
    partitions = assertNumberPartitions(3, lc, dag.getMeta(collector));
    Assert.assertTrue("container reused", lc.dnmgr.getPhysicalPlan().getContainers().containsAll(containers));
    // check deployment
    for (PTOperator p : partitions) {
        StramTestSupport.waitForActivation(lc, p);
    }
    PartitionLoadWatch.remove(splitPartition);
    for (PTContainer container : lc.dnmgr.getPhysicalPlan().getContainers()) {
        int memory = 0;
        for (PTOperator operator : container.getOperators()) {
            memory += operator.getBufferServerMemory();
            memory += operator.getOperatorMeta().getValue(OperatorContext.MEMORY_MB);
        }
        Assert.assertEquals("memory", memory, container.getRequiredMemoryMB());
    }
    PTOperator planInput = lc.findByLogicalNode(dag.getMeta(input));
    LocalStreamingContainer c = StramTestSupport.waitForActivation(lc, planInput);
    Map<Integer, Node<?>> nodeMap = c.getNodes();
    Assert.assertEquals("number operators " + nodeMap, 1, nodeMap.size());
    @SuppressWarnings({ "unchecked" }) TestInputOperator<Integer> inputDeployed = (TestInputOperator<Integer>) nodeMap.get(planInput.getId()).getOperator();
    Assert.assertNotNull("" + nodeMap, inputDeployed);
    // add tuple that matches the partition key and check that each partition receives it
    ArrayList<Integer> inputTuples = new ArrayList<>();
    LOG.debug("Number of partitions {}", partitions.size());
    for (PTOperator p : partitions) {
        // default partitioning has one port mapping with a single partition key
        LOG.debug("Partition key map size: {}", p.getPartitionKeys().size());
        inputTuples.add(p.getPartitionKeys().values().iterator().next().partitions.iterator().next());
    }
    inputDeployed.testTuples = Collections.synchronizedList(new ArrayList<List<Integer>>());
    inputDeployed.testTuples.add(inputTuples);
    for (PTOperator p : partitions) {
        Integer expectedTuple = p.getPartitionKeys().values().iterator().next().partitions.iterator().next();
        List<Object> receivedTuples;
        int i = 0;
        while ((receivedTuples = CollectorOperator.receivedTuples.get(collector.prefix + p.getId())) == null || receivedTuples.isEmpty()) {
            if (i++ % 100 == 0) {
                LOG.debug("Waiting for tuple: " + p);
            }
            sleep(10);
        }
        Assert.assertEquals("received " + p, Arrays.asList(expectedTuple), receivedTuples);
    }
    // single output operator to receive tuple from each partition
    List<PTOperator> operators = lc.getPlanOperators(dag.getMeta(singleCollector));
    Assert.assertEquals("number output operator instances " + operators, 1, operators.size());
    // ensure redeploy
    StramTestSupport.waitForActivation(lc, operators.get(0));
    List<Object> receivedTuples;
    while ((receivedTuples = CollectorOperator.receivedTuples.get(singleCollector.prefix + operators.get(0).getId())) == null || receivedTuples.size() < inputTuples.size()) {
        LOG.debug("Waiting for tuple: " + operators.get(0) + " expected: " + inputTuples + " received: " + receivedTuples);
        sleep(20);
    }
    Assert.assertEquals("output tuples " + receivedTuples, Sets.newHashSet(inputTuples), Sets.newHashSet(receivedTuples));
    lc.shutdown();
}

Example 17 with StatsListener

use of com.datatorrent.api.StatsListener in project apex-core by apache.

the class StatsTest method baseTestForQueueSize.

@SuppressWarnings("SleepWhileInLoop")
private void baseTestForQueueSize(int maxTuples, TestCollectorStatsListener statsListener, DAG.Locality locality) throws Exception {
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/baseTestForQueueSize").getAbsolutePath();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 200);
    TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
    testOper.setMaxTuples(maxTuples);
    TestCollector collector = dag.addOperator("Collector", new TestCollector());
    if (statsListener != null) {
        dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { statsListener }));
    }
    dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(locality);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    StreamingContainerManager dnmgr = lc.getStreamingContainerManager();
    Map<Integer, PTOperator> operatorMap = dnmgr.getPhysicalPlan().getAllOperators();
    for (PTOperator p : operatorMap.values()) {
        StramTestSupport.waitForActivation(lc, p);
    }
    long startTms = System.currentTimeMillis();
    if (statsListener != null) {
        while (statsListener.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
            Thread.sleep(300);
            LOG.debug("Waiting for stats");
        }
    } else {
        while (collector.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
            Thread.sleep(300);
            LOG.debug("Waiting for stats");
        }
    }
    if (statsListener != null) {
        statsListener.validateStats();
    } else {
        collector.validateStats();
    }
    lc.shutdown();
}

Example 18 with StatsListener

use of com.datatorrent.api.StatsListener in project apex-core by apache.

the class AutoMetricTest method testMetricPropagation.

/**
   * Verify custom stats generated by operator are propagated and trigger repartition.
   *
   * @throws Exception
   */
@Test
@SuppressWarnings("SleepWhileInLoop")
public void testMetricPropagation() throws Exception {
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
    TestStatsListener sl = new TestStatsListener();
    dag.setOperatorAttribute(testOper, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) sl));
    GenericTestOperator collector = dag.addOperator("Collector", new GenericTestOperator());
    dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(Locality.CONTAINER_LOCAL);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    long startTms = System.currentTimeMillis();
    while (TestOperator.lastMetric == null && StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
        Thread.sleep(300);
        LOG.debug("Waiting for stats");
    }
    while (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
        if (sl.lastPropVal) {
            break;
        }
        Thread.sleep(100);
        LOG.debug("Waiting for property set");
    }
    lc.shutdown();
    Assert.assertNotNull("metric received", TestOperator.lastMetric);
    Assert.assertEquals("metric message", "interesting", TestOperator.lastMetric.message);
    Assert.assertTrue("attribute defined stats listener called", TestOperator.lastMetric.attributeListenerCalled);
    Assert.assertSame("single thread", TestOperator.definePartitionsThread, TestOperator.processStatsThread);
    Assert.assertTrue("property set", sl.lastPropVal);
}

Example 19 with StatsListener

use of com.datatorrent.api.StatsListener in project apex-core by apache.

the class PhysicalPlanTest method testAugmentedDynamicPartitioning.

/**
   * Test covering scenario when only new partitions are added during dynamic partitioning and there
   * are no changes to existing partitions and partition mapping
   */
@Test
public void testAugmentedDynamicPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new TestAugmentingPartitioner<TestGeneratorInputOperator>(3));
    dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) new PartitioningTest.PartitionLoadWatch()));
    OperatorMeta o1Meta = dag.getMeta(o1);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    OperatorMeta o2Meta = dag.getMeta(o2);
    dag.addStream("o1.outport1", o1.outport, o2.inport1);
    int maxContainers = 10;
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    Assert.assertEquals("number of containers", 4, plan.getContainers().size());
    List<PTOperator> o1ops = plan.getOperators(o1Meta);
    Assert.assertEquals("number of o1 operators", 3, o1ops.size());
    List<PTOperator> o2ops = plan.getOperators(o2Meta);
    Assert.assertEquals("number of o2 operators", 1, o2ops.size());
    Set<PTOperator> expUndeploy = Sets.newLinkedHashSet();
    expUndeploy.addAll(plan.getOperators(o2Meta));
    expUndeploy.add(plan.getOperators(o2Meta).get(0).upstreamMerge.values().iterator().next());
    for (int i = 0; i < 2; ++i) {
        PartitioningTest.PartitionLoadWatch.put(o1ops.get(i), 1);
        plan.onStatusUpdate(o1ops.get(i));
    }
    ctx.backupRequests = 0;
    ctx.events.remove(0).run();
    Assert.assertEquals("number of containers", 6, plan.getContainers().size());
    Assert.assertEquals("undeployed opertors", expUndeploy, ctx.undeploy);
}

Example 20 with StatsListener

use of com.datatorrent.api.StatsListener in project apex-core by apache.

the class PhysicalPlanTest method testSingleFinalMxNPartitioning.

/**
   * MxN partitioning. When source and sink of a stream are partitioned, a
   * separate unifier is created container local with each downstream partition.
   */
@Test
public void testSingleFinalMxNPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<TestGeneratorInputOperator>(2));
    dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) new PartitioningTest.PartitionLoadWatch()));
    dag.setOutputPortAttribute(o1.outport, PortContext.UNIFIER_SINGLE_FINAL, true);
    OperatorMeta o1Meta = dag.getMeta(o1);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<TestGeneratorInputOperator>(3));
    dag.setOperatorAttribute(o2, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
    OperatorMeta o2Meta = dag.getMeta(o2);
    dag.addStream("o1.outport1", o1.outport, o2.inport1);
    int maxContainers = 10;
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    Assert.assertEquals("number of containers", 6, plan.getContainers().size());
    List<PTOperator> inputOperators = new ArrayList<>();
    for (int i = 0; i < 2; i++) {
        PTContainer container = plan.getContainers().get(i);
        Assert.assertEquals("number operators " + container, 1, container.getOperators().size());
        Assert.assertEquals("operators " + container, o1Meta.getName(), container.getOperators().get(0).getOperatorMeta().getName());
        inputOperators.add(container.getOperators().get(0));
    }
    PTOperator inputUnifier = null;
    {
        PTContainer container = plan.getContainers().get(2);
        Assert.assertEquals("number operators " + container, 1, container.getOperators().size());
        PTOperator pUnifier = container.getOperators().get(0);
        Assert.assertEquals("operators " + container, o1Meta.getMeta(o1.outport).getUnifierMeta().getName(), pUnifier.getOperatorMeta().getName());
        Assert.assertTrue("single unifier " + pUnifier, pUnifier.isUnifier());
        Assert.assertEquals("" + pUnifier, 2, pUnifier.getInputs().size());
        for (int inputIndex = 0; inputIndex < pUnifier.getInputs().size(); inputIndex++) {
            PTInput input = pUnifier.getInputs().get(inputIndex);
            Assert.assertEquals("source port name " + pUnifier, "outport", input.source.portName);
            Assert.assertEquals("" + pUnifier, inputOperators.get(inputIndex), input.source.source);
            Assert.assertEquals("partition keys " + input.partitions, null, input.partitions);
        }
        Assert.assertEquals("number outputs " + pUnifier, 1, pUnifier.getOutputs().size());
        PTOutput output = pUnifier.getOutputs().get(0);
        Assert.assertEquals("number inputs " + output, 3, output.sinks.size());
        for (int inputIndex = 0; inputIndex < output.sinks.size(); ++inputIndex) {
            Assert.assertEquals("output sink " + output, o2Meta.getName(), output.sinks.get(inputIndex).target.getName());
            Assert.assertEquals("destination port name " + output, GenericTestOperator.IPORT1, output.sinks.get(inputIndex).portName);
        }
        inputUnifier = pUnifier;
    }
    List<Integer> partitionKeySizes = new ArrayList<>();
    for (int i = 3; i < 6; i++) {
        PTContainer container = plan.getContainers().get(i);
        Assert.assertEquals("number operators " + container, 1, container.getOperators().size());
        Assert.assertEquals("operators " + container, o2Meta.getName(), container.getOperators().get(0).getOperatorMeta().getName());
        PTOperator operator = container.getOperators().get(0);
        Assert.assertEquals("operators " + container, o2Meta.getName(), operator.getOperatorMeta().getName());
        Assert.assertEquals("number inputs " + operator, 1, operator.getInputs().size());
        PTInput input = operator.getInputs().get(0);
        Assert.assertEquals("" + operator, inputUnifier, input.source.source);
        Assert.assertNotNull("input partitions " + operator, input.partitions);
        partitionKeySizes.add(input.partitions.partitions.size());
    }
    Assert.assertEquals("input partition sizes count", 3, partitionKeySizes.size());
    Collections.sort(partitionKeySizes);
    Assert.assertEquals("input partition sizes", Arrays.asList(1, 1, 2), partitionKeySizes);
    // scale down N from 3 to 2 and then from 2 to 1
    for (int i = 0; i < 2; i++) {
        List<PTOperator> ptos = plan.getOperators(o2Meta);
        Set<PTOperator> expUndeploy = Sets.newHashSet(ptos);
        for (PTOperator ptOperator : ptos) {
            //expUndeploy.addAll(ptOperator.upstreamMerge.values());
            expUndeploy.add(ptOperator);
            PartitioningTest.PartitionLoadWatch.put(ptOperator, -1);
            plan.onStatusUpdate(ptOperator);
        }
        ctx.backupRequests = 0;
        ctx.events.remove(0).run();
        Assert.assertEquals("single unifier ", 1, plan.getMergeOperators(o1Meta).size());
        Set<PTOperator> expDeploy = Sets.newHashSet(plan.getOperators(o2Meta));
        // The unifier and o2 operators are expected to be deployed because of partition key changes
        for (PTOperator ptOperator : plan.getOperators(o2Meta)) {
            expDeploy.add(ptOperator);
        }
        // from 3 to 2 the containers decrease from 5 to 4, but from 2 to 1 the container remains same because single unifier are not inline with single operator partition
        Assert.assertEquals("number of containers", 5 - i, plan.getContainers().size());
        Assert.assertEquals("number of operators", 2 - i, plan.getOperators(o2Meta).size());
        Assert.assertEquals("undeployed operators " + ctx.undeploy, expUndeploy, ctx.undeploy);
        Assert.assertEquals("deployed operators " + ctx.deploy, expDeploy, ctx.deploy);
    }
    // scale up N from 1 to 2 and then from 2 to 3
    for (int i = 0; i < 2; i++) {
        List<PTOperator> unChangedOps = new LinkedList<>(plan.getOperators(o2Meta));
        PTOperator o2p1 = unChangedOps.remove(0);
        Set<PTOperator> expUndeploy = Sets.newHashSet(o2p1);
        PartitioningTest.PartitionLoadWatch.put(o2p1, 1);
        plan.onStatusUpdate(o2p1);
        Assert.assertEquals("repartition event", 1, ctx.events.size());
        ctx.backupRequests = 0;
        ctx.events.remove(0).run();
        Assert.assertEquals("single unifier ", 1, plan.getMergeOperators(o1Meta).size());
        Assert.assertEquals("N partitions after scale up " + o2Meta, 2 + i, plan.getOperators(o2Meta).size());
        for (PTOperator o : plan.getOperators(o2Meta)) {
            Assert.assertNotNull(o.container);
            Assert.assertEquals("number operators ", 1, o.container.getOperators().size());
        }
        Set<PTOperator> expDeploy = Sets.newHashSet(plan.getOperators(o2Meta));
        expDeploy.removeAll(unChangedOps);
        Assert.assertEquals("number of containers", 5 + i, plan.getContainers().size());
        Assert.assertEquals("undeployed operators" + ctx.undeploy, expUndeploy, ctx.undeploy);
        Assert.assertEquals("deployed operators" + ctx.deploy, expDeploy, ctx.deploy);
    }
    // scale down M to 1
    {
        Set<PTOperator> expUndeploy = Sets.newHashSet();
        Set<PTOperator> expDeploy = Sets.newHashSet();
        expUndeploy.addAll(plan.getMergeOperators(o1Meta));
        for (PTOperator o2p : plan.getOperators(o2Meta)) {
            expUndeploy.add(o2p);
            expDeploy.add(o2p);
        }
        for (PTOperator o1p : plan.getOperators(o1Meta)) {
            expUndeploy.add(o1p);
            PartitioningTest.PartitionLoadWatch.put(o1p, -1);
            plan.onStatusUpdate(o1p);
        }
        Assert.assertEquals("repartition event", 1, ctx.events.size());
        ctx.events.remove(0).run();
        Assert.assertEquals("M partitions after scale down " + o1Meta, 1, plan.getOperators(o1Meta).size());
        expUndeploy.removeAll(plan.getOperators(o1Meta));
        Assert.assertEquals("undeploy", expUndeploy, ctx.undeploy);
        Assert.assertEquals("deploy", expDeploy, ctx.deploy);
    }
    // scale up M to 2
    Assert.assertEquals("M partitions " + o1Meta, 1, plan.getOperators(o1Meta).size());
    {
        Set<PTOperator> expUndeploy = Sets.newHashSet();
        Set<PTOperator> expDeploy = Sets.newHashSet();
        for (PTOperator o1p : plan.getOperators(o1Meta)) {
            expUndeploy.add(o1p);
            PartitioningTest.PartitionLoadWatch.put(o1p, 1);
            plan.onStatusUpdate(o1p);
        }
        Assert.assertEquals("repartition event", 1, ctx.events.size());
        ctx.events.remove(0).run();
        Assert.assertEquals("M partitions after scale up " + o1Meta, 2, plan.getOperators(o1Meta).size());
        expDeploy.addAll(plan.getOperators(o1Meta));
        expDeploy.addAll(plan.getMergeOperators(o1Meta));
        for (PTOperator o2p : plan.getOperators(o2Meta)) {
            expUndeploy.add(o2p);
            expDeploy.add(o2p);
            Assert.assertNotNull(o2p.container);
            Assert.assertEquals("number operators ", 1, o2p.container.getOperators().size());
        }
        Assert.assertEquals("undeploy", expUndeploy, ctx.undeploy);
        Assert.assertEquals("deploy", expDeploy, ctx.deploy);
    }
}