Search in sources :

Example 6 with PhysicalPlan

use of com.datatorrent.stram.plan.physical.PhysicalPlan in project apex-core by apache.

the class OutputUnifiedTest method testManyToOnePartition.

@Test
public void testManyToOnePartition() throws Exception {
    TestInputOperator i1 = new TestInputOperator();
    dag.addOperator("i1", i1);
    GenericTestOperator op1 = new GenericTestOperator();
    dag.addOperator("op1", op1);
    dag.setOperatorAttribute(op1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
    TestOutputOperator op2 = new TestOutputOperator();
    dag.addOperator("op2", op2);
    dag.addStream("s1", i1.output, op1.inport1);
    dag.addStream("s2", op1.outport1, op2.inport);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan physicalPlan = scm.getPhysicalPlan();
    List<PTContainer> containers = physicalPlan.getContainers();
    Assert.assertEquals("Number of containers", 5, containers.size());
    assignContainers(scm, containers);
    testOutputAttribute(dag, i1, scm, physicalPlan, false);
    testOutputAttribute(dag, op1, scm, physicalPlan, true);
}
Also used : PhysicalPlan(com.datatorrent.stram.plan.physical.PhysicalPlan) GenericTestOperator(com.datatorrent.stram.engine.GenericTestOperator) PTContainer(com.datatorrent.stram.plan.physical.PTContainer) TestOutputOperator(com.datatorrent.stram.engine.TestOutputOperator) TestInputOperator(com.datatorrent.stram.stream.OiOEndWindowTest.TestInputOperator) Test(org.junit.Test)

Example 7 with PhysicalPlan

use of com.datatorrent.stram.plan.physical.PhysicalPlan in project apex-core by apache.

the class OutputUnifiedTest method testMxNPartition.

@Test
public void testMxNPartition() throws Exception {
    TestInputOperator i1 = new TestInputOperator();
    dag.addOperator("i1", i1);
    GenericTestOperator op1 = new GenericTestOperator();
    dag.addOperator("op1", op1);
    dag.setOperatorAttribute(op1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
    TestOutputOperator op2 = new TestOutputOperator();
    dag.addOperator("op2", op2);
    dag.setOperatorAttribute(op2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    dag.addStream("s1", i1.output, op1.inport1);
    dag.addStream("s2", op1.outport1, op2.inport);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan physicalPlan = scm.getPhysicalPlan();
    List<PTContainer> containers = physicalPlan.getContainers();
    Assert.assertEquals("Number of containers", 6, containers.size());
    assignContainers(scm, containers);
    testOutputAttribute(dag, i1, scm, physicalPlan, false);
    testOutputAttribute(dag, op1, scm, physicalPlan, true);
}
Also used : PhysicalPlan(com.datatorrent.stram.plan.physical.PhysicalPlan) GenericTestOperator(com.datatorrent.stram.engine.GenericTestOperator) PTContainer(com.datatorrent.stram.plan.physical.PTContainer) TestOutputOperator(com.datatorrent.stram.engine.TestOutputOperator) TestInputOperator(com.datatorrent.stram.stream.OiOEndWindowTest.TestInputOperator) Test(org.junit.Test)

Example 8 with PhysicalPlan

use of com.datatorrent.stram.plan.physical.PhysicalPlan in project apex-core by apache.

the class GenericOperatorPropertyCodecTest method testGenericOperatorPropertyCodec.

@Test
public void testGenericOperatorPropertyCodec() {
    LogicalPlan dag = new LogicalPlan();
    Map<Class<?>, Class<? extends StringCodec<?>>> codecs = new HashMap<>();
    codecs.put(GenericOperatorProperty.class, GenericOperatorProperty.GenericOperatorPropertyStringCodec.class);
    dag.setAttribute(com.datatorrent.api.Context.DAGContext.STRING_CODECS, codecs);
    dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    OperatorMeta o1Meta = dag.getMeta(o1);
    TestPlanContext ctx = new TestPlanContext();
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    ctx.deploy.clear();
    ctx.undeploy.clear();
    PlanModifier pm = new PlanModifier(plan);
    try {
        pm.setOperatorProperty(o1Meta.getName(), "genericOperatorProperty", "xyz");
        // cannot set properties on an operator that is already deployed.
        Assert.fail("validation error expected");
    } catch (javax.validation.ValidationException e) {
        Assert.assertTrue(e.getMessage().contains(o1Meta.toString()));
    }
    GenericTestOperator newOperator = new GenericTestOperator();
    pm.addOperator("newOperator", newOperator);
    pm.setOperatorProperty("newOperator", "genericOperatorProperty", "xyz");
    Assert.assertEquals("", "xyz", newOperator.getGenericOperatorProperty().obtainString());
}
Also used : PhysicalPlan(com.datatorrent.stram.plan.physical.PhysicalPlan) OperatorMeta(com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta) HashMap(java.util.HashMap) PlanModifier(com.datatorrent.stram.plan.physical.PlanModifier) StringCodec(com.datatorrent.api.StringCodec) GenericOperatorProperty(com.datatorrent.stram.engine.GenericOperatorProperty) GenericTestOperator(com.datatorrent.stram.engine.GenericTestOperator) TestPlanContext(com.datatorrent.stram.plan.TestPlanContext) MemoryStorageAgent(com.datatorrent.stram.support.StramTestSupport.MemoryStorageAgent) LogicalPlan(com.datatorrent.stram.plan.logical.LogicalPlan) Test(org.junit.Test)

Example 9 with PhysicalPlan

use of com.datatorrent.stram.plan.physical.PhysicalPlan in project apex-core by apache.

the class LogicalPlanModificationTest method testRemoveOperator2.

@Test
public void testRemoveOperator2() {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    OperatorMeta o1Meta = dag.getMeta(o1);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    OperatorMeta o2Meta = dag.getMeta(o2);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    OperatorMeta o3Meta = dag.getMeta(o3);
    LogicalPlan.StreamMeta s1 = dag.addStream("o1.outport1", o1.outport1, o2.inport1, o3.inport1).setLocality(Locality.CONTAINER_LOCAL);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    ctx.deploy.clear();
    ctx.undeploy.clear();
    Assert.assertEquals("containers " + plan.getContainers(), 1, plan.getContainers().size());
    Assert.assertEquals("physical operators " + plan.getAllOperators(), 3, plan.getAllOperators().size());
    Assert.assertEquals("sinks s1 " + s1.getSinks(), 2, s1.getSinks().size());
    List<PTOperator> o2PhysicalOpers = plan.getOperators(o2Meta);
    Assert.assertEquals("instances " + o2Meta, 1, o2PhysicalOpers.size());
    PlanModifier pm = new PlanModifier(plan);
    // remove operator w/o removing the stream
    pm.removeOperator(o2Meta.getName());
    pm.applyChanges(ctx);
    Assert.assertEquals("sinks s1 " + s1.getSinks(), 1, s1.getSinks().size());
    Assert.assertTrue("undeploy " + ctx.undeploy, ctx.undeploy.containsAll(o2PhysicalOpers));
    Set<PTOperator> expDeploy = Sets.newHashSet();
    // TODO: container local operators should be included in undeploy/deploy
    //expDeploy.addAll(plan.getOperators(o1Meta));
    //expDeploy.addAll(plan.getOperators(o3Meta));
    Assert.assertEquals("deploy " + ctx.deploy, ctx.deploy, expDeploy);
    Assert.assertEquals("streams " + dag.getAllStreams(), 1, dag.getAllStreams().size());
    Assert.assertEquals("operators " + dag.getAllOperators(), 2, dag.getAllOperators().size());
    Assert.assertTrue("operators " + dag.getAllOperators(), dag.getAllOperators().containsAll(Sets.newHashSet(o1Meta, o3Meta)));
}
Also used : PhysicalPlan(com.datatorrent.stram.plan.physical.PhysicalPlan) OperatorMeta(com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta) PTOperator(com.datatorrent.stram.plan.physical.PTOperator) PlanModifier(com.datatorrent.stram.plan.physical.PlanModifier) GenericTestOperator(com.datatorrent.stram.engine.GenericTestOperator) TestPlanContext(com.datatorrent.stram.plan.TestPlanContext) LogicalPlan(com.datatorrent.stram.plan.logical.LogicalPlan) Test(org.junit.Test) PhysicalPlanTest(com.datatorrent.stram.plan.physical.PhysicalPlanTest)

Example 10 with PhysicalPlan

use of com.datatorrent.stram.plan.physical.PhysicalPlan in project apex-core by apache.

the class LogicalPlanModificationTest method testRemoveStream.

@Test
public void testRemoveStream() {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    PlanModifier pm = new PlanModifier(plan);
    pm.removeStream("o1.outport1");
    pm.applyChanges(ctx);
    Assert.assertEquals("undeploy " + ctx.undeploy, 2, ctx.undeploy.size());
    Assert.assertEquals("deploy " + ctx.deploy, 2, ctx.deploy.size());
}
Also used : PhysicalPlan(com.datatorrent.stram.plan.physical.PhysicalPlan) PlanModifier(com.datatorrent.stram.plan.physical.PlanModifier) GenericTestOperator(com.datatorrent.stram.engine.GenericTestOperator) TestPlanContext(com.datatorrent.stram.plan.TestPlanContext) Test(org.junit.Test) PhysicalPlanTest(com.datatorrent.stram.plan.physical.PhysicalPlanTest)

Aggregations

PhysicalPlan (com.datatorrent.stram.plan.physical.PhysicalPlan)57 GenericTestOperator (com.datatorrent.stram.engine.GenericTestOperator)47 Test (org.junit.Test)47 PTOperator (com.datatorrent.stram.plan.physical.PTOperator)38 PTContainer (com.datatorrent.stram.plan.physical.PTContainer)34 LogicalPlan (com.datatorrent.stram.plan.logical.LogicalPlan)23 StramTestSupport (com.datatorrent.stram.support.StramTestSupport)18 MemoryStorageAgent (com.datatorrent.stram.support.StramTestSupport.MemoryStorageAgent)18 PhysicalPlanTest (com.datatorrent.stram.plan.physical.PhysicalPlanTest)17 OperatorDeployInfo (com.datatorrent.stram.api.OperatorDeployInfo)16 Checkpoint (com.datatorrent.stram.api.Checkpoint)14 TestPlanContext (com.datatorrent.stram.plan.TestPlanContext)13 OperatorMeta (com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta)13 PlanModifier (com.datatorrent.stram.plan.physical.PlanModifier)9 TestGeneratorInputOperator (com.datatorrent.stram.engine.TestGeneratorInputOperator)7 Operator (com.datatorrent.api.Operator)5 UpdateCheckpointsContext (com.datatorrent.stram.StreamingContainerManager.UpdateCheckpointsContext)5 AsyncFSStorageAgent (com.datatorrent.common.util.AsyncFSStorageAgent)4 StatsListener (com.datatorrent.api.StatsListener)3 FSStorageAgent (com.datatorrent.common.util.FSStorageAgent)3