Examples with TestPlanContext com.datatorrent.stram.plan.TestPlanContext used on opensource projects

Example 36 with TestPlanContext

use of com.datatorrent.stram.plan.TestPlanContext in project apex-core by apache.

the class PhysicalPlanTest method testStaticPartitioning.

@Test
public void testStaticPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    TestGeneratorInputOperator node0 = dag.addOperator("node0", TestGeneratorInputOperator.class);
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    PartitioningTestOperator partitioned = dag.addOperator("partitioned", PartitioningTestOperator.class);
    partitioned.setPartitionCount(partitioned.partitionKeys.length);
    GenericTestOperator singleton1 = dag.addOperator("singleton1", GenericTestOperator.class);
    GenericTestOperator singleton2 = dag.addOperator("singleton2", GenericTestOperator.class);
    dag.addStream("n0.inport1", node0.outport, node1.inport1);
    dag.addStream("n1.outport1", node1.outport1, partitioned.inport1, partitioned.inportWithCodec);
    dag.addStream("mergeStream", partitioned.outport1, singleton1.inport1, singleton2.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    OperatorMeta partitionedMeta = dag.getMeta(partitioned);
    dag.validate();
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 2, plan.getContainers().size());
    Assert.assertNotNull("partition map", partitioned.partitions);
    Assert.assertEquals("partition map " + partitioned.partitions, 3, partitioned.partitions.size());
    List<PTOperator> n2Instances = plan.getOperators(partitionedMeta);
    Assert.assertEquals("partition instances " + n2Instances, partitioned.partitionKeys.length, n2Instances.size());
    for (int i = 0; i < n2Instances.size(); i++) {
        PTOperator po = n2Instances.get(i);
        Map<String, PTInput> inputsMap = new HashMap<>();
        for (PTInput input : po.getInputs()) {
            inputsMap.put(input.portName, input);
            Assert.assertEquals("partitions " + input, Sets.newHashSet(partitioned.partitionKeys[i]), input.partitions.partitions);
        // Assert.assertEquals("codec " + input.logicalStream, PartitioningTestStreamCodec.class, input.logicalStream.getCodecClass());
        }
        Assert.assertEquals("number inputs " + inputsMap, Sets.newHashSet(PartitioningTestOperator.IPORT1, PartitioningTestOperator.INPORT_WITH_CODEC), inputsMap.keySet());
    }
    Collection<PTOperator> unifiers = plan.getMergeOperators(partitionedMeta);
    Assert.assertEquals("number unifiers " + partitionedMeta, 0, unifiers.size());
}

Example 37 with TestPlanContext

use of com.datatorrent.stram.plan.TestPlanContext in project apex-core by apache.

the class PhysicalPlanTest method testNumberOfUnifiers.

@Test
public void testNumberOfUnifiers() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    dag.addStream("node1.outport1", node1.outport1, node2.inport1);
    dag.setOperatorAttribute(node1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(5));
    dag.setOutputPortAttribute(node1.outport1, PortContext.UNIFIER_LIMIT, 3);
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    List<PTContainer> containers = plan.getContainers();
    int unifierCount = 0;
    int totalOperators = 0;
    for (PTContainer container : containers) {
        List<PTOperator> operators = container.getOperators();
        for (PTOperator operator : operators) {
            totalOperators++;
            if (operator.isUnifier()) {
                unifierCount++;
            }
        }
    }
    Assert.assertEquals("Number of operators", 8, totalOperators);
    Assert.assertEquals("Number of unifiers", 2, unifierCount);
}

Example 38 with TestPlanContext

use of com.datatorrent.stram.plan.TestPlanContext in project apex-core by apache.

the class PhysicalPlanTest method testParallelPartitioning.

@Test
public void testParallelPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.addStream("o1Output1", o1.outport1, o2.inport1, o3.inport1).setLocality(null);
    dag.addStream("o2Output1", o2.outport1, o3.inport2).setLocality(Locality.CONTAINER_LOCAL);
    dag.setInputPortAttribute(o3.inport2, PortContext.PARTITION_PARALLEL, true);
    // parallel partition two downstream operators
    PartitioningTestOperator o3_1 = dag.addOperator("o3_1", PartitioningTestOperator.class);
    o3_1.fixedCapacity = false;
    dag.setInputPortAttribute(o3_1.inport1, PortContext.PARTITION_PARALLEL, true);
    OperatorMeta o3_1Meta = dag.getMeta(o3_1);
    GenericTestOperator o3_2 = dag.addOperator("o3_2", GenericTestOperator.class);
    dag.setInputPortAttribute(o3_2.inport1, PortContext.PARTITION_PARALLEL, true);
    OperatorMeta o3_2Meta = dag.getMeta(o3_2);
    dag.addStream("o3outport1", o3.outport1, o3_1.inport1, o3_2.inport1).setLocality(Locality.CONTAINER_LOCAL);
    // join within parallel partition
    GenericTestOperator o4 = dag.addOperator("o4", GenericTestOperator.class);
    dag.setInputPortAttribute(o4.inport1, PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(o4.inport2, PortContext.PARTITION_PARALLEL, true);
    OperatorMeta o4Meta = dag.getMeta(o4);
    dag.addStream("o3_1.outport1", o3_1.outport1, o4.inport1).setLocality(Locality.CONTAINER_LOCAL);
    dag.addStream("o3_2.outport1", o3_2.outport1, o4.inport2).setLocality(Locality.CONTAINER_LOCAL);
    // non inline
    GenericTestOperator o5single = dag.addOperator("o5single", GenericTestOperator.class);
    dag.addStream("o4outport1", o4.outport1, o5single.inport1);
    int maxContainers = 4;
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new TestPlanContext());
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 4, plan.getContainers().size());
    PTContainer container1 = plan.getContainers().get(0);
    Assert.assertEquals("number operators " + container1, 1, container1.getOperators().size());
    Assert.assertEquals("operators " + container1, "o1", container1.getOperators().get(0).getOperatorMeta().getName());
    for (int i = 1; i < 3; i++) {
        PTContainer container2 = plan.getContainers().get(i);
        Assert.assertEquals("number operators " + container2, 5, container2.getOperators().size());
        Set<String> expectedLogicalNames = Sets.newHashSet("o2", "o3", o3_1Meta.getName(), o3_2Meta.getName(), o4Meta.getName());
        Set<String> actualLogicalNames = Sets.newHashSet();
        for (PTOperator p : container2.getOperators()) {
            actualLogicalNames.add(p.getOperatorMeta().getName());
        }
        Assert.assertEquals("operator names " + container2, expectedLogicalNames, actualLogicalNames);
    }
    List<OperatorMeta> inlineOperators = Lists.newArrayList(dag.getMeta(o2), o3_1Meta, o3_2Meta);
    for (OperatorMeta ow : inlineOperators) {
        List<PTOperator> partitionedInstances = plan.getOperators(ow);
        Assert.assertEquals("" + partitionedInstances, 2, partitionedInstances.size());
        for (PTOperator p : partitionedInstances) {
            Assert.assertEquals("outputs " + p, 1, p.getOutputs().size());
            Assert.assertTrue("downstream inline " + p.getOutputs().get(0), p.getOutputs().get(0).isDownStreamInline());
        }
    }
    // container 4: Unifier for o4 & O5
    PTContainer container4 = plan.getContainers().get(3);
    PTOperator ptOperatorO5 = plan.getOperators(dag.getMeta(o5single)).get(0);
    PTOperator unifier = ptOperatorO5.upstreamMerge.values().iterator().next();
    Assert.assertEquals("number operators " + container4, 2, container4.getOperators().size());
    Assert.assertEquals("operators " + container4, o4Meta.getMeta(o4.outport1).getUnifierMeta(), unifier.getOperatorMeta());
    Assert.assertEquals("unifier inputs" + unifier.getInputs(), 2, unifier.getInputs().size());
    Assert.assertEquals("unifier outputs" + unifier.getOutputs(), 1, unifier.getOutputs().size());
    OperatorMeta o5Meta = dag.getMeta(o5single);
    Assert.assertEquals("operators " + container4, o5Meta, ptOperatorO5.getOperatorMeta());
    List<PTOperator> o5Instances = plan.getOperators(o5Meta);
    Assert.assertEquals("" + o5Instances, 1, o5Instances.size());
    Assert.assertEquals("inputs" + ptOperatorO5.getInputs(), 1, ptOperatorO5.getInputs().size());
    Assert.assertEquals("inputs" + ptOperatorO5.getInputs(), unifier, ptOperatorO5.getInputs().get(0).source.source);
    // verify partitioner was called for parallel partition
    Assert.assertNotNull("partitioner called " + o3_1, o3_1.partitions);
    for (PTOperator p : plan.getOperators(o3_1Meta)) {
        Assert.assertEquals("inputs " + p, 1, p.getInputs().size());
        for (PTInput pti : p.getInputs()) {
            Assert.assertNull("partition keys " + pti, pti.partitions);
        }
    }
}

Example 39 with TestPlanContext

use of com.datatorrent.stram.plan.TestPlanContext in project apex-core by apache.

the class PhysicalPlanTest method testInlineMultipleInputs.

@Test
public void testInlineMultipleInputs() {
    LogicalPlan dag = new LogicalPlan();
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    dag.addStream("n1Output1", node1.outport1, node3.inport1).setLocality(Locality.CONTAINER_LOCAL);
    dag.addStream("n2Output1", node2.outport1, node3.inport2).setLocality(Locality.CONTAINER_LOCAL);
    int maxContainers = 5;
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new TestPlanContext());
    PhysicalPlan deployer = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 1, deployer.getContainers().size());
    PTOutput node1Out = deployer.getOperators(dag.getMeta(node1)).get(0).getOutputs().get(0);
    Assert.assertTrue("inline " + node1Out, node1Out.isDownStreamInline());
    // per current logic, different container is assigned to second input node
    PTOutput node2Out = deployer.getOperators(dag.getMeta(node2)).get(0).getOutputs().get(0);
    Assert.assertTrue("inline " + node2Out, node2Out.isDownStreamInline());
}

Example 40 with TestPlanContext

use of com.datatorrent.stram.plan.TestPlanContext in project apex-core by apache.

the class LogicalPlanModificationTest method testNewOperatorRecoveryWindowIds.

@Test
public void testNewOperatorRecoveryWindowIds() {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    ctx.deploy.clear();
    ctx.undeploy.clear();
    LogicalPlan.OperatorMeta o1Meta = dag.getMeta(o1);
    List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
    PhysicalPlanTest.setActivationCheckpoint(o1Partitions.get(0), 10);
    PlanModifier pm = new PlanModifier(plan);
    GenericTestOperator o2 = new GenericTestOperator();
    GenericTestOperator o3 = new GenericTestOperator();
    pm.addOperator("o2", o2);
    pm.addOperator("o3", o3);
    pm.addStream("s1", o1.outport1, o2.inport2);
    pm.addStream("s2", o2.outport1, o3.inport1);
    pm.applyChanges(ctx);
    LogicalPlan.OperatorMeta o2Meta = plan.getLogicalPlan().getMeta(o2);
    List<PTOperator> o2Partitions = plan.getOperators(o2Meta);
    Assert.assertEquals("o2 activation checkpoint " + o2Meta, 10, o2Partitions.get(0).getRecoveryCheckpoint().windowId);
    LogicalPlan.OperatorMeta o3Meta = plan.getLogicalPlan().getMeta(o3);
    List<PTOperator> o3Partitions = plan.getOperators(o3Meta);
    Assert.assertEquals("o3 activation checkpoint " + o2Meta, 10, o3Partitions.get(0).getRecoveryCheckpoint().windowId);
}