Examples with OperatorMeta com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta used on opensource projects

Example 56 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlanTest method testSingleFinalUnifierMultiInput.

@Test
public void testSingleFinalUnifierMultiInput() {
    LogicalPlan dag = new LogicalPlan();
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
    OperatorMeta o1Meta = dag.getMeta(o1);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(4));
    dag.setInputPortAttribute(o2.inport1, PortContext.UNIFIER_SINGLE_FINAL, true);
    OperatorMeta o2Meta = dag.getMeta(o2);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.setOperatorAttribute(o3, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    OperatorMeta o3Meta = dag.getMeta(o3);
    dag.addStream("o1o2o3", o1.outport1, o2.inport1, o3.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 12);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    Assert.assertEquals("number of containers", 10, plan.getContainers().size());
    Assert.assertEquals("o1 merge unifiers", 1, plan.getMergeOperators(o1Meta).size());
    // Check the merge unifier
    {
        PTContainer container = plan.getContainers().get(3);
        Assert.assertEquals("number of operators " + container, 1, container.getOperators().size());
        PTOperator operator = container.getOperators().get(0);
        Assert.assertTrue("unifier check " + operator, operator.isUnifier());
        Assert.assertEquals("operator meta " + operator, o1Meta.getMeta(o1.outport1).getUnifierMeta(), operator.getOperatorMeta());
    }
    int numberO2Containers = 0;
    int numberO3Containers = 0;
    Set<String> expectedNames = Sets.newHashSet(o1Meta.getMeta(o1.outport1).getUnifierMeta().getName(), o3Meta.getName());
    for (int i = 4; i < 10; i++) {
        PTContainer container = plan.getContainers().get(i);
        List<PTOperator> operators = container.getOperators();
        Assert.assertTrue("expected operator count " + container, (operators.size() <= 2) && (operators.size() > 0));
        if (operators.size() == 1) {
            Assert.assertEquals("operator in container " + container, o2Meta, operators.get(0).getOperatorMeta());
            ++numberO2Containers;
        } else if (operators.size() == 2) {
            Set<String> names = Sets.newHashSet();
            for (PTOperator operator : container.getOperators()) {
                names.add(operator.getOperatorMeta().getName());
            }
            Assert.assertEquals("container operators " + container, expectedNames, names);
            ++numberO3Containers;
        }
    }
    Assert.assertEquals("number o2 containers", 4, numberO2Containers);
    Assert.assertEquals("number o3 containers", 2, numberO3Containers);
}

Example 57 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlanTest method testStaticPartitioning.

@Test
public void testStaticPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    TestGeneratorInputOperator node0 = dag.addOperator("node0", TestGeneratorInputOperator.class);
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    PartitioningTestOperator partitioned = dag.addOperator("partitioned", PartitioningTestOperator.class);
    partitioned.setPartitionCount(partitioned.partitionKeys.length);
    GenericTestOperator singleton1 = dag.addOperator("singleton1", GenericTestOperator.class);
    GenericTestOperator singleton2 = dag.addOperator("singleton2", GenericTestOperator.class);
    dag.addStream("n0.inport1", node0.outport, node1.inport1);
    dag.addStream("n1.outport1", node1.outport1, partitioned.inport1, partitioned.inportWithCodec);
    dag.addStream("mergeStream", partitioned.outport1, singleton1.inport1, singleton2.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    OperatorMeta partitionedMeta = dag.getMeta(partitioned);
    dag.validate();
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 2, plan.getContainers().size());
    Assert.assertNotNull("partition map", partitioned.partitions);
    Assert.assertEquals("partition map " + partitioned.partitions, 3, partitioned.partitions.size());
    List<PTOperator> n2Instances = plan.getOperators(partitionedMeta);
    Assert.assertEquals("partition instances " + n2Instances, partitioned.partitionKeys.length, n2Instances.size());
    for (int i = 0; i < n2Instances.size(); i++) {
        PTOperator po = n2Instances.get(i);
        Map<String, PTInput> inputsMap = new HashMap<>();
        for (PTInput input : po.getInputs()) {
            inputsMap.put(input.portName, input);
            Assert.assertEquals("partitions " + input, Sets.newHashSet(partitioned.partitionKeys[i]), input.partitions.partitions);
        //Assert.assertEquals("codec " + input.logicalStream, PartitioningTestStreamCodec.class, input.logicalStream.getCodecClass());
        }
        Assert.assertEquals("number inputs " + inputsMap, Sets.newHashSet(PartitioningTestOperator.IPORT1, PartitioningTestOperator.INPORT_WITH_CODEC), inputsMap.keySet());
    }
    Collection<PTOperator> unifiers = plan.getMergeOperators(partitionedMeta);
    Assert.assertEquals("number unifiers " + partitionedMeta, 0, unifiers.size());
}

Example 58 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlanTest method testSingleFinalUnifierInputOverride.

@Test
public void testSingleFinalUnifierInputOverride() {
    LogicalPlan dag = new LogicalPlan();
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
    OperatorMeta o1Meta = dag.getMeta(o1);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    dag.setInputPortAttribute(o2.inport1, PortContext.UNIFIER_SINGLE_FINAL, true);
    OperatorMeta o2Meta = dag.getMeta(o2);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 10);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    Assert.assertEquals("number of containers", 6, plan.getContainers().size());
    Assert.assertEquals("o1 merge unifiers", 1, plan.getMergeOperators(o1Meta).size());
    dag.setOutputPortAttribute(o1.outport1, PortContext.UNIFIER_SINGLE_FINAL, false);
    ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    plan = new PhysicalPlan(dag, ctx);
    Assert.assertEquals("number of containers", 6, plan.getContainers().size());
    Assert.assertEquals("o1 merge unifiers", 1, plan.getMergeOperators(o1Meta).size());
    dag.setOutputPortAttribute(o1.outport1, PortContext.UNIFIER_SINGLE_FINAL, true);
    dag.setInputPortAttribute(o2.inport1, PortContext.UNIFIER_SINGLE_FINAL, false);
    ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    plan = new PhysicalPlan(dag, ctx);
    Assert.assertEquals("number of containers", 5, plan.getContainers().size());
    Set<String> expectedNames = Sets.newHashSet(o1Meta.getMeta(o1.outport1).getUnifierMeta().getName(), o2Meta.getName());
    for (int i = 3; i < 5; ++i) {
        PTContainer container = plan.getContainers().get(i);
        Assert.assertEquals("o2 container size", 2, container.getOperators().size());
        Set<String> names = Sets.newHashSet();
        for (PTOperator operator : container.getOperators()) {
            names.add(operator.getOperatorMeta().getName());
        }
        Assert.assertEquals("o2 container operators", expectedNames, names);
    }
}

Example 59 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlanTest method testRepartitioningScaleDownSinglePartition.

/**
   * Test unifier gets removed when number partitions drops to 1.
   */
@Test
public void testRepartitioningScaleDownSinglePartition() {
    LogicalPlan dag = new LogicalPlan();
    TestInputOperator<?> o1 = dag.addOperator("o1", TestInputOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("o1.outport1", o1.output, o2.inport1);
    OperatorMeta o1Meta = dag.getMeta(o1);
    dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
    Assert.assertEquals("partitions " + o1Partitions, 2, o1Partitions.size());
    PTOperator o1p1 = o1Partitions.get(0);
    PTOperator p1Doper = o1p1.getOutputs().get(0).sinks.get(0).target;
    Assert.assertSame("", p1Doper.getOperatorMeta(), o1Meta.getMeta(o1.output).getUnifierMeta());
    Assert.assertTrue("unifier ", p1Doper.isUnifier());
    Assert.assertEquals("Unifiers " + o1Meta, 1, o1p1.getOutputs().get(0).sinks.size());
    Collection<PTOperator> o1Unifiers = new ArrayList<>(plan.getOperators(dag.getMeta(o2)).get(0).upstreamMerge.values());
    StatsListener l = o1p1.statsListeners.get(0);
    Assert.assertTrue("stats handlers " + o1p1.statsListeners, l instanceof PartitioningTest.PartitionLoadWatch);
    PartitioningTest.PartitionLoadWatch.put(o1p1, -1);
    PartitioningTest.PartitionLoadWatch.put(o1Partitions.get(1), -1);
    plan.onStatusUpdate(o1p1);
    plan.onStatusUpdate(o1Partitions.get(1));
    Assert.assertEquals("partition scaling triggered", 1, ctx.events.size());
    ctx.events.remove(0).run();
    List<PTOperator> o1NewPartitions = plan.getOperators(o1Meta);
    Assert.assertEquals("partitions " + o1NewPartitions, 1, o1NewPartitions.size());
    List<PTOperator> o1NewUnifiers = new ArrayList<>(plan.getOperators(dag.getMeta(o2)).get(0).upstreamMerge.values());
    Assert.assertEquals("unifiers " + o1Meta, 0, o1NewUnifiers.size());
    p1Doper = o1p1.getOutputs().get(0).sinks.get(0).target;
    Assert.assertTrue("", p1Doper.getOperatorMeta() == dag.getMeta(o2));
    Assert.assertFalse("unifier ", p1Doper.isUnifier());
    Assert.assertTrue("removed unifier from deployment " + ctx.undeploy, ctx.undeploy.containsAll(o1Unifiers));
    Assert.assertFalse("removed unifier from deployment " + ctx.deploy, ctx.deploy.containsAll(o1Unifiers));
    // scale up, ensure unifier is setup at activation checkpoint
    setActivationCheckpoint(o1NewPartitions.get(0), 3);
    PartitioningTest.PartitionLoadWatch.put(o1NewPartitions.get(0), 1);
    plan.onStatusUpdate(o1NewPartitions.get(0));
    Assert.assertEquals("partition scaling triggered", 1, ctx.events.size());
    ctx.events.remove(0).run();
    o1NewUnifiers.addAll(plan.getOperators(dag.getMeta(o2)).get(0).upstreamMerge.values());
    Assert.assertEquals("unifiers " + o1Meta, 1, o1NewUnifiers.size());
    Assert.assertEquals("unifier activation checkpoint " + o1Meta, 3, o1NewUnifiers.get(0).recoveryCheckpoint.windowId);
}

Example 60 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class PhysicalPlanTest method testSingleFinalCascadingUnifier.

@Test
public void testSingleFinalCascadingUnifier() {
    LogicalPlan dag = new LogicalPlan();
    //TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    PartitioningTestOperator o1 = dag.addOperator("o1", PartitioningTestOperator.class);
    o1.partitionKeys = new Integer[] { 0, 1, 2, 3 };
    o1.setPartitionCount(3);
    dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
    dag.setOutputPortAttribute(o1.outport1, PortContext.UNIFIER_LIMIT, 2);
    dag.setOutputPortAttribute(o1.outport1, PortContext.UNIFIER_SINGLE_FINAL, true);
    OperatorMeta o1Meta = dag.getMeta(o1);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
    OperatorMeta o2Meta = dag.getMeta(o2);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 12);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    Assert.assertEquals("number of containers", 10, plan.getContainers().size());
    List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
    Assert.assertEquals("partitions " + o1Meta, 4, o1Partitions.size());
    Assert.assertEquals("partitioned map " + o1.partitions, 4, o1.partitions.size());
    List<PTOperator> o2Partitions = plan.getOperators(o2Meta);
    Assert.assertEquals("partitions " + o1Meta, 3, o2Partitions.size());
    for (PTOperator o : o1Partitions) {
        Assert.assertEquals("outputs " + o, 1, o.getOutputs().size());
        for (PTOutput out : o.getOutputs()) {
            Assert.assertEquals("sinks " + out, 1, out.sinks.size());
        }
        Assert.assertNotNull("container " + o, o.getContainer());
    }
    List<PTOperator> o1Unifiers = plan.getMergeOperators(o1Meta);
    // 2 cascadingUnifiers and one-downstream partition unifier
    Assert.assertEquals("o1Unifiers " + o1Meta, 3, o1Unifiers.size());
    List<PTOperator> finalUnifiers = new ArrayList<>();
    for (PTOperator o : o1Unifiers) {
        Assert.assertEquals("inputs " + o, 2, o.getInputs().size());
        Assert.assertEquals("outputs " + o, 1, o.getOutputs().size());
        List<PTInput> sinks = o.getOutputs().get(0).sinks;
        boolean finalUnifier = sinks.size() > 0 ? (sinks.get(0).target.getOperatorMeta() == o2Meta) : false;
        if (!finalUnifier) {
            for (PTOutput out : o.getOutputs()) {
                Assert.assertEquals("sinks " + out, 1, out.sinks.size());
                Assert.assertTrue(out.sinks.get(0).target.isUnifier());
            }
        } else {
            for (PTOutput out : o.getOutputs()) {
                Assert.assertEquals("sinks " + out, 3, out.sinks.size());
                for (PTInput in : out.sinks) {
                    Assert.assertFalse(in.target.isUnifier());
                }
            }
            finalUnifiers.add(o);
        }
        Assert.assertNotNull("container " + o, o.getContainer());
    }
    Assert.assertEquals("o1 final unifiers", 1, finalUnifiers.size());
    for (int i = 0; i < 4; i++) {
        PTContainer container = plan.getContainers().get(i);
        Assert.assertEquals("number operators " + container, 1, container.getOperators().size());
        Assert.assertTrue(o1Partitions.contains(container.getOperators().get(0)));
    }
    for (int i = 4; i < 7; i++) {
        PTContainer container = plan.getContainers().get(i);
        Assert.assertEquals("number operators " + container, 1, container.getOperators().size());
        Assert.assertTrue(o1Unifiers.contains(container.getOperators().get(0)));
    }
    for (int i = 7; i < 10; i++) {
        PTContainer container = plan.getContainers().get(i);
        Assert.assertEquals("number operators " + container, 1, container.getOperators().size());
        Assert.assertTrue(o2Partitions.contains(container.getOperators().get(0)));
    }
    PTOperator p1 = o1Partitions.get(0);
    StatsListener l = p1.statsListeners.get(0);
    Assert.assertTrue("stats handlers " + p1.statsListeners, l instanceof PartitioningTest.PartitionLoadWatch);
    PartitioningTest.PartitionLoadWatch.put(p1, 1);
    plan.onStatusUpdate(p1);
    Assert.assertEquals("partition scaling triggered", 1, ctx.events.size());
    o1.partitionKeys = new Integer[] { 0, 1, 2, 3, 4 };
    ctx.events.remove(0).run();
    o1Partitions = plan.getOperators(o1Meta);
    Assert.assertEquals("partitions " + o1Meta, 5, o1Partitions.size());
    Assert.assertEquals("partitioned map " + o1.partitions, 5, o1.partitions.size());
    o1Unifiers = plan.getMergeOperators(o1Meta);
    // 3(l1)x2(l2)
    Assert.assertEquals("o1Unifiers " + o1Meta, 4, o1Unifiers.size());
    for (PTOperator o : o1Unifiers) {
        Assert.assertNotNull("container null: " + o, o.getContainer());
    }
}