Examples with LogicalPlan com.datatorrent.stram.plan.logical.LogicalPlan used on opensource projects

Example 1 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class PhysicalPlanTest method testDefaultPartitionerWithParallel.

@Test
public void testDefaultPartitionerWithParallel() throws InterruptedException {
    final MutableInt loadInd = new MutableInt();
    StatsListener listener = new StatsListener() {

        @Override
        public Response processStats(BatchedOperatorStats stats) {
            Response response = new Response();
            response.repartitionRequired = true;
            response.loadIndicator = loadInd.intValue();
            return response;
        }
    };
    LogicalPlan dag = new LogicalPlan();
    GenericTestOperator nodeX = dag.addOperator("X", GenericTestOperator.class);
    dag.setOperatorAttribute(nodeX, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    dag.setOperatorAttribute(nodeX, Context.OperatorContext.STATS_LISTENERS, Lists.newArrayList(listener));
    GenericTestOperator nodeY = dag.addOperator("Y", GenericTestOperator.class);
    dag.setOperatorAttribute(nodeY, Context.OperatorContext.PARTITIONER, new TestPartitioner<GenericTestOperator>());
    GenericTestOperator nodeZ = dag.addOperator("Z", GenericTestOperator.class);
    dag.addStream("Stream1", nodeX.outport1, nodeY.inport1, nodeZ.inport1);
    dag.addStream("Stream2", nodeX.outport2, nodeY.inport2, nodeZ.inport2);
    dag.setInputPortAttribute(nodeY.inport1, Context.PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(nodeY.inport2, Context.PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(nodeZ.inport1, Context.PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(nodeZ.inport2, Context.PortContext.PARTITION_PARALLEL, true);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    TestPlanContext ctx = new TestPlanContext();
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    LogicalPlan.OperatorMeta metaOfX = dag.getMeta(nodeX);
    LogicalPlan.OperatorMeta metaOfY = dag.getMeta(nodeY);
    Assert.assertEquals("number operators " + metaOfX.getName(), 2, plan.getOperators(metaOfX).size());
    Assert.assertEquals("number operators " + metaOfY.getName(), 2, plan.getOperators(metaOfY).size());
    List<PTOperator> ptOfX = plan.getOperators(metaOfX);
    for (PTOperator physicalX : ptOfX) {
        Assert.assertEquals("2 streams " + physicalX.getOutputs(), 2, physicalX.getOutputs().size());
        for (PTOutput outputPort : physicalX.getOutputs()) {
            Set<PTOperator> dopers = Sets.newHashSet();
            Assert.assertEquals("sink of " + metaOfX.getName() + " id " + physicalX.id + " port " + outputPort.portName, 2, outputPort.sinks.size());
            for (PTInput inputPort : outputPort.sinks) {
                dopers.add(inputPort.target);
            }
            Assert.assertEquals(2, dopers.size());
        }
    }
    // Invoke redo-partition of PhysicalPlan, no partition change
    loadInd.setValue(0);
    for (PTOperator ptOperator : ptOfX) {
        plan.onStatusUpdate(ptOperator);
    }
    ctx.events.remove(0).run();
    for (PTOperator physicalX : ptOfX) {
        Assert.assertEquals("2 streams " + physicalX.getOutputs(), 2, physicalX.getOutputs().size());
        for (PTOutput outputPort : physicalX.getOutputs()) {
            Set<PTOperator> dopers = Sets.newHashSet();
            Assert.assertEquals("sink of " + metaOfX.getName() + " id " + physicalX.id + " port " + outputPort.portName, 2, outputPort.sinks.size());
            for (PTInput inputPort : outputPort.sinks) {
                dopers.add(inputPort.target);
            }
            Assert.assertEquals(2, dopers.size());
        }
    }
    // scale up by splitting first partition
    loadInd.setValue(1);
    plan.onStatusUpdate(ptOfX.get(0));
    ctx.events.get(0).run();
    List<PTOperator> ptOfXScaleUp = plan.getOperators(metaOfX);
    Assert.assertEquals("3 partitons " + ptOfXScaleUp, 3, ptOfXScaleUp.size());
    for (PTOperator physicalX : ptOfXScaleUp) {
        Assert.assertEquals("2 streams " + physicalX.getOutputs(), 2, physicalX.getOutputs().size());
        for (PTOutput outputPort : physicalX.getOutputs()) {
            Set<PTOperator> dopers = Sets.newHashSet();
            Assert.assertEquals("sink of " + metaOfX.getName() + " id " + physicalX.id + " port " + outputPort.portName, 2, outputPort.sinks.size());
            for (PTInput inputPort : outputPort.sinks) {
                dopers.add(inputPort.target);
            }
            Assert.assertEquals(2, dopers.size());
        }
    }
}

Example 2 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class PhysicalPlanTest method testStatsListenerContextWrappers.

/**
 * Test that internally all stats listeners are handled through StatsListenerWithContext.
 * Following cases are tested
 *
 * Operator implementing StatsListener
 * Operator implementing StatsListenerWithContext
 * Operator with STATS_LISTENERS attribute set to StatsListener
 * Operator with STATS_LISTENERS attribute set to StatsListenerWithContext
 */
@Test
public void testStatsListenerContextWrappers() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    StatsListenerOperator o1 = dag.addOperator("o1", new StatsListenerOperator());
    GenericTestOperator o2 = dag.addOperator("o2", new GenericTestOperator());
    dag.setAttribute(o2, OperatorContext.STATS_LISTENERS, Lists.<StatsListener>newArrayList(mock(StatsListener.class)));
    GenericTestOperator o3 = dag.addOperator("o3", new GenericTestOperator());
    dag.setAttribute(o3, OperatorContext.STATS_LISTENERS, Lists.<StatsListener>newArrayList(mock(StatsListenerWithContext.class)));
    StatsListenerOperatorOld o4 = dag.addOperator("o4", new StatsListenerOperatorOld());
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    PTOperator p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    StatsListener l = p1.statsListeners.get(0);
    Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);
    PTOperator p2 = plan.getOperators(dag.getMeta(o2)).get(0);
    l = p1.statsListeners.get(0);
    Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);
    PTOperator p3 = plan.getOperators(dag.getMeta(o3)).get(0);
    l = p1.statsListeners.get(0);
    Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);
    PTOperator p4 = plan.getOperators(dag.getMeta(o4)).get(0);
    l = p1.statsListeners.get(0);
    Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);
}

Example 3 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class PhysicalPlanTest method testContainerCores.

@Test
public void testContainerCores() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    GenericTestOperator o4 = dag.addOperator("o4", GenericTestOperator.class);
    GenericTestOperator o5 = dag.addOperator("o5", GenericTestOperator.class);
    GenericTestOperator o6 = dag.addOperator("o6", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.VCORES, 1);
    dag.setOperatorAttribute(o2, OperatorContext.VCORES, 2);
    dag.setOperatorAttribute(o3, OperatorContext.VCORES, 3);
    dag.setOperatorAttribute(o4, OperatorContext.VCORES, 4);
    dag.setOperatorAttribute(o5, OperatorContext.VCORES, 5);
    dag.setOperatorAttribute(o6, OperatorContext.VCORES, 6);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1).setLocality(Locality.CONTAINER_LOCAL);
    dag.addStream("o2.outport1", o2.outport1, o3.inport1, o4.inport1).setLocality(Locality.THREAD_LOCAL);
    dag.addStream("o3.output1", o3.outport1, o5.inport1).setLocality(Locality.THREAD_LOCAL);
    dag.addStream("o4.output1", o4.outport1, o5.inport2).setLocality(Locality.THREAD_LOCAL);
    dag.addStream("o5.output1", o5.outport1, o6.inport1).setLocality(Locality.CONTAINER_LOCAL);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 1, plan.getContainers().size());
    Assert.assertEquals("vcores container 1 is 12", 12, plan.getContainers().get(0).getRequiredVCores());
}

Example 4 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class PhysicalPlanTest method testCascadingUnifier.

@Test
public void testCascadingUnifier() {
    LogicalPlan dag = new LogicalPlan();
    PartitioningTestOperator o1 = dag.addOperator("o1", PartitioningTestOperator.class);
    o1.partitionKeys = new Integer[] { 0, 1, 2, 3 };
    o1.setPartitionCount(o1.partitionKeys.length);
    dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
    dag.setOutputPortAttribute(o1.outport1, PortContext.UNIFIER_LIMIT, 2);
    OperatorMeta o1Meta = dag.getMeta(o1);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
    OperatorMeta o2Meta = dag.getMeta(o2);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 10);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    Assert.assertEquals("number of containers", 9, plan.getContainers().size());
    List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
    Assert.assertEquals("partitions " + o1Meta, 4, o1Partitions.size());
    Assert.assertEquals("partitioned map " + o1.partitions, 4, o1.partitions.size());
    List<PTOperator> o2Partitions = plan.getOperators(o2Meta);
    Assert.assertEquals("partitions " + o1Meta, 3, o2Partitions.size());
    for (PTOperator o : o1Partitions) {
        Assert.assertEquals("outputs " + o, 1, o.getOutputs().size());
        for (PTOutput out : o.getOutputs()) {
            Assert.assertEquals("sinks " + out, 1, out.sinks.size());
        }
        Assert.assertNotNull("container " + o, o.getContainer());
    }
    List<PTOperator> o1Unifiers = plan.getMergeOperators(o1Meta);
    // 2 cascadingUnifiers to per-downstream partition unifier(s)
    Assert.assertEquals("o1Unifiers " + o1Meta, 2, o1Unifiers.size());
    for (PTOperator o : o1Unifiers) {
        Assert.assertEquals("inputs " + o, 2, o.getInputs().size());
        Assert.assertEquals("outputs " + o, 1, o.getOutputs().size());
        for (PTOutput out : o.getOutputs()) {
            Assert.assertEquals("sinks " + out, 3, out.sinks.size());
            for (PTInput in : out.sinks) {
                // MxN unifier
                Assert.assertTrue(in.target.isUnifier());
                Assert.assertEquals(1, in.target.getOutputs().get(0).sinks.size());
            }
        }
        Assert.assertNotNull("container " + o, o.getContainer());
    }
    for (int i = 0; i < 4; i++) {
        PTContainer container = plan.getContainers().get(i);
        Assert.assertEquals("number operators " + container, 1, container.getOperators().size());
        Assert.assertTrue(o1Partitions.contains(container.getOperators().get(0)));
    }
    for (int i = 4; i < 6; i++) {
        PTContainer container = plan.getContainers().get(i);
        Assert.assertEquals("number operators " + container, 1, container.getOperators().size());
        Assert.assertTrue(o1Unifiers.contains(container.getOperators().get(0)));
    }
    for (int i = 6; i < 9; i++) {
        PTContainer container = plan.getContainers().get(i);
        Assert.assertEquals("number operators " + container, 2, container.getOperators().size());
        Assert.assertTrue(o2Partitions.contains(container.getOperators().get(0)));
    }
    PTOperator p1 = o1Partitions.get(0);
    StatsListener l = p1.statsListeners.get(0);
    Assert.assertTrue("stats handlers " + p1.statsListeners, l instanceof PartitioningTest.PartitionLoadWatch);
    PartitioningTest.PartitionLoadWatch.put(p1, 1);
    plan.onStatusUpdate(p1);
    Assert.assertEquals("partition scaling triggered", 1, ctx.events.size());
    o1.partitionKeys = new Integer[] { 0, 1, 2, 3, 4 };
    ctx.events.remove(0).run();
    o1Partitions = plan.getOperators(o1Meta);
    Assert.assertEquals("partitions " + o1Meta, 5, o1Partitions.size());
    Assert.assertEquals("partitioned map " + o1.partitions, 5, o1.partitions.size());
    o1Unifiers = plan.getMergeOperators(o1Meta);
    // 3(l1)x2(l2)
    Assert.assertEquals("o1Unifiers " + o1Meta, 3, o1Unifiers.size());
    for (PTOperator o : o1Unifiers) {
        Assert.assertNotNull("container null: " + o, o.getContainer());
    }
}

Example 5 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class PhysicalPlanTest method testContainersForSlidingWindow.

@Test
public void testContainersForSlidingWindow() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.APPLICATION_WINDOW_COUNT, 4);
    dag.setOperatorAttribute(o1, OperatorContext.SLIDE_BY_WINDOW_COUNT, 2);
    dag.getOperatorMeta("o1").getMeta(o1.outport1).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB, 2000);
    dag.getOperatorMeta("o1").getMeta(o1.outport2).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB, 4000);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.addStream("o1.outport2", o1.outport2, o2.inport2);
    dag.addStream("o2.outport1", o2.outport1, o3.inport1);
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 5, plan.getContainers().size());
    boolean sawOutput1Slider = false;
    boolean sawOutput2Slider = false;
    for (PTContainer container : plan.getContainers()) {
        Assert.assertEquals("number of operators in each container is 1", container.operators.size(), 1);
        if (container.operators.get(0).isUnifier()) {
            String name = container.operators.get(0).getName();
            if (name.equals("o1.outport1#slider")) {
                sawOutput1Slider = true;
                Assert.assertEquals("container memory is 2512", container.getRequiredMemoryMB(), 2512);
            } else if (name.equals("o1.outport2#slider")) {
                sawOutput2Slider = true;
                Assert.assertEquals("container memory is 2512", container.getRequiredMemoryMB(), 4512);
            }
        }
    }
    Assert.assertEquals("Found output1 slider", true, sawOutput1Slider);
    Assert.assertEquals("Found output2 slider", true, sawOutput2Slider);
}