Examples with TestPlanContext com.datatorrent.stram.plan.TestPlanContext used on opensource projects

Example 31 with TestPlanContext

use of com.datatorrent.stram.plan.TestPlanContext in project apex-core by apache.

the class PhysicalPlanTest method testParallelPartitionForSlidingWindow.

@Test
public void testParallelPartitionForSlidingWindow() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.SLIDE_BY_WINDOW_COUNT, 2);
    dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<>(2));
    dag.setInputPortAttribute(o2.inport1, PortContext.PARTITION_PARALLEL, true);
    dag.setOperatorAttribute(o1, OperatorContext.APPLICATION_WINDOW_COUNT, 4);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.addStream("o2.outport1", o2.outport1, o3.inport1);
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 7, plan.getContainers().size());
}

Example 32 with TestPlanContext

use of com.datatorrent.stram.plan.TestPlanContext in project apex-core by apache.

the class PhysicalPlanTest method testDefaultPartitioning.

@Test
public void testDefaultPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    dag.addStream("node1.outport1", node1.outport1, node2.inport2, node2.inport1);
    int initialPartitionCount = 5;
    OperatorMeta node2Decl = dag.getMeta(node2);
    node2Decl.getAttributes().put(OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(initialPartitionCount));
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    List<PTOperator> n2Instances = plan.getOperators(node2Decl);
    Assert.assertEquals("partition instances " + n2Instances, initialPartitionCount, n2Instances.size());
    List<Integer> assignedPartitionKeys = Lists.newArrayList();
    for (int i = 0; i < n2Instances.size(); i++) {
        PTOperator n2Partition = n2Instances.get(i);
        Assert.assertNotNull("partition keys null: " + n2Partition, n2Partition.getPartitionKeys());
        Map<InputPort<?>, PartitionKeys> pkeys = n2Partition.getPartitionKeys();
        // one port partitioned
        Assert.assertEquals("partition keys size: " + pkeys, 1, pkeys.size());
        InputPort<?> expectedPort = node2.inport2;
        Assert.assertEquals("partition port: " + pkeys, expectedPort, pkeys.keySet().iterator().next());
        Assert.assertEquals("partition mask: " + pkeys, "111", Integer.toBinaryString(pkeys.get(expectedPort).mask));
        Set<Integer> pks = pkeys.get(expectedPort).partitions;
        Assert.assertTrue("number partition keys: " + pkeys, pks.size() == 1 || pks.size() == 2);
        assignedPartitionKeys.addAll(pks);
    }
    int expectedMask = Integer.parseInt("111", 2);
    Assert.assertEquals("assigned partitions ", expectedMask + 1, assignedPartitionKeys.size());
    for (int i = 0; i <= expectedMask; i++) {
        Assert.assertTrue("" + assignedPartitionKeys, assignedPartitionKeys.contains(i));
    }
}

Example 33 with TestPlanContext

use of com.datatorrent.stram.plan.TestPlanContext in project apex-core by apache.

the class PhysicalPlanTest method testNumberOfUnifiersWithEvenPartitions.

@Test
public void testNumberOfUnifiersWithEvenPartitions() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    dag.addStream("node1.outport1", node1.outport1, node2.inport1);
    dag.setOperatorAttribute(node1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(8));
    dag.setOutputPortAttribute(node1.outport1, PortContext.UNIFIER_LIMIT, 4);
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    List<PTContainer> containers = plan.getContainers();
    int unifierCount = 0;
    int totalOperators = 0;
    for (PTContainer container : containers) {
        List<PTOperator> operators = container.getOperators();
        for (PTOperator operator : operators) {
            totalOperators++;
            if (operator.isUnifier()) {
                unifierCount++;
            }
        }
    }
    Assert.assertEquals("Number of operators", 12, totalOperators);
    Assert.assertEquals("Number of unifiers", 3, unifierCount);
}

Example 34 with TestPlanContext

use of com.datatorrent.stram.plan.TestPlanContext in project apex-core by apache.

the class PhysicalPlanTest method testMxNPartitionForSlidingWindow.

@Test
public void testMxNPartitionForSlidingWindow() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.APPLICATION_WINDOW_COUNT, 4);
    dag.setOperatorAttribute(o1, OperatorContext.SLIDE_BY_WINDOW_COUNT, 2);
    dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<>(2));
    dag.getOperatorMeta("o1").getMeta(o1.outport1).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB, 1024);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<>(2));
    dag.setOperatorAttribute(o2, OperatorContext.SLIDE_BY_WINDOW_COUNT, 2);
    dag.setOperatorAttribute(o2, OperatorContext.APPLICATION_WINDOW_COUNT, 4);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.addStream("o2.outport1", o2.outport1, o3.inport1);
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 9, plan.getContainers().size());
}

Example 35 with TestPlanContext

use of com.datatorrent.stram.plan.TestPlanContext in project apex-core by apache.

the class PhysicalPlanTest method testContainerSize.

@Test
public void testContainerSize() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.VCORES, 1);
    dag.setOperatorAttribute(o2, OperatorContext.VCORES, 2);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.addStream("o2.outport1", o2.outport1, o3.inport1);
    dag.setOperatorAttribute(o2, OperatorContext.MEMORY_MB, 4000);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 2, plan.getContainers().size());
    Assert.assertEquals("memory container 1", 2560, plan.getContainers().get(0).getRequiredMemoryMB());
    Assert.assertEquals("vcores container 1", 1, plan.getContainers().get(0).getRequiredVCores());
    Assert.assertEquals("memory container 2", 4512, plan.getContainers().get(1).getRequiredMemoryMB());
    Assert.assertEquals("vcores container 2", 2, plan.getContainers().get(1).getRequiredVCores());
    Assert.assertEquals("number of operators in container 1", 2, plan.getContainers().get(0).getOperators().size());
}