use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.
the class PhysicalPlanTest method testDefaultPartitionerWithParallel.
@Test
public void testDefaultPartitionerWithParallel() throws InterruptedException {
final MutableInt loadInd = new MutableInt();
StatsListener listener = new StatsListener() {
@Override
public Response processStats(BatchedOperatorStats stats) {
Response response = new Response();
response.repartitionRequired = true;
response.loadIndicator = loadInd.intValue();
return response;
}
};
LogicalPlan dag = new LogicalPlan();
GenericTestOperator nodeX = dag.addOperator("X", GenericTestOperator.class);
dag.setOperatorAttribute(nodeX, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
dag.setOperatorAttribute(nodeX, Context.OperatorContext.STATS_LISTENERS, Lists.newArrayList(listener));
GenericTestOperator nodeY = dag.addOperator("Y", GenericTestOperator.class);
dag.setOperatorAttribute(nodeY, Context.OperatorContext.PARTITIONER, new TestPartitioner<GenericTestOperator>());
GenericTestOperator nodeZ = dag.addOperator("Z", GenericTestOperator.class);
dag.addStream("Stream1", nodeX.outport1, nodeY.inport1, nodeZ.inport1);
dag.addStream("Stream2", nodeX.outport2, nodeY.inport2, nodeZ.inport2);
dag.setInputPortAttribute(nodeY.inport1, Context.PortContext.PARTITION_PARALLEL, true);
dag.setInputPortAttribute(nodeY.inport2, Context.PortContext.PARTITION_PARALLEL, true);
dag.setInputPortAttribute(nodeZ.inport1, Context.PortContext.PARTITION_PARALLEL, true);
dag.setInputPortAttribute(nodeZ.inport2, Context.PortContext.PARTITION_PARALLEL, true);
StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
TestPlanContext ctx = new TestPlanContext();
PhysicalPlan plan = new PhysicalPlan(dag, ctx);
LogicalPlan.OperatorMeta metaOfX = dag.getMeta(nodeX);
LogicalPlan.OperatorMeta metaOfY = dag.getMeta(nodeY);
Assert.assertEquals("number operators " + metaOfX.getName(), 2, plan.getOperators(metaOfX).size());
Assert.assertEquals("number operators " + metaOfY.getName(), 2, plan.getOperators(metaOfY).size());
List<PTOperator> ptOfX = plan.getOperators(metaOfX);
for (PTOperator physicalX : ptOfX) {
Assert.assertEquals("2 streams " + physicalX.getOutputs(), 2, physicalX.getOutputs().size());
for (PTOutput outputPort : physicalX.getOutputs()) {
Set<PTOperator> dopers = Sets.newHashSet();
Assert.assertEquals("sink of " + metaOfX.getName() + " id " + physicalX.id + " port " + outputPort.portName, 2, outputPort.sinks.size());
for (PTInput inputPort : outputPort.sinks) {
dopers.add(inputPort.target);
}
Assert.assertEquals(2, dopers.size());
}
}
// Invoke redo-partition of PhysicalPlan, no partition change
loadInd.setValue(0);
for (PTOperator ptOperator : ptOfX) {
plan.onStatusUpdate(ptOperator);
}
ctx.events.remove(0).run();
for (PTOperator physicalX : ptOfX) {
Assert.assertEquals("2 streams " + physicalX.getOutputs(), 2, physicalX.getOutputs().size());
for (PTOutput outputPort : physicalX.getOutputs()) {
Set<PTOperator> dopers = Sets.newHashSet();
Assert.assertEquals("sink of " + metaOfX.getName() + " id " + physicalX.id + " port " + outputPort.portName, 2, outputPort.sinks.size());
for (PTInput inputPort : outputPort.sinks) {
dopers.add(inputPort.target);
}
Assert.assertEquals(2, dopers.size());
}
}
// scale up by splitting first partition
loadInd.setValue(1);
plan.onStatusUpdate(ptOfX.get(0));
ctx.events.get(0).run();
List<PTOperator> ptOfXScaleUp = plan.getOperators(metaOfX);
Assert.assertEquals("3 partitons " + ptOfXScaleUp, 3, ptOfXScaleUp.size());
for (PTOperator physicalX : ptOfXScaleUp) {
Assert.assertEquals("2 streams " + physicalX.getOutputs(), 2, physicalX.getOutputs().size());
for (PTOutput outputPort : physicalX.getOutputs()) {
Set<PTOperator> dopers = Sets.newHashSet();
Assert.assertEquals("sink of " + metaOfX.getName() + " id " + physicalX.id + " port " + outputPort.portName, 2, outputPort.sinks.size());
for (PTInput inputPort : outputPort.sinks) {
dopers.add(inputPort.target);
}
Assert.assertEquals(2, dopers.size());
}
}
}
use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.
the class LogicalPlanModificationTest method testRemoveStream.
@Test
public void testRemoveStream() {
GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
dag.addStream("o1.outport1", o1.outport1, o2.inport1);
TestPlanContext ctx = new TestPlanContext();
dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
PhysicalPlan plan = new PhysicalPlan(dag, ctx);
PlanModifier pm = new PlanModifier(plan);
pm.removeStream("o1.outport1");
pm.applyChanges(ctx);
Assert.assertEquals("undeploy " + ctx.undeploy, 2, ctx.undeploy.size());
Assert.assertEquals("deploy " + ctx.deploy, 2, ctx.deploy.size());
}
use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.
the class LogicalPlanModificationTest method testRemoveOperator2.
@Test
public void testRemoveOperator2() {
GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
OperatorMeta o1Meta = dag.getMeta(o1);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
OperatorMeta o2Meta = dag.getMeta(o2);
GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
OperatorMeta o3Meta = dag.getMeta(o3);
LogicalPlan.StreamMeta s1 = dag.addStream("o1.outport1", o1.outport1, o2.inport1, o3.inport1).setLocality(Locality.CONTAINER_LOCAL);
TestPlanContext ctx = new TestPlanContext();
dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
PhysicalPlan plan = new PhysicalPlan(dag, ctx);
ctx.deploy.clear();
ctx.undeploy.clear();
Assert.assertEquals("containers " + plan.getContainers(), 1, plan.getContainers().size());
Assert.assertEquals("physical operators " + plan.getAllOperators(), 3, plan.getAllOperators().size());
Assert.assertEquals("sinks s1 " + s1.getSinks(), 2, s1.getSinks().size());
List<PTOperator> o2PhysicalOpers = plan.getOperators(o2Meta);
Assert.assertEquals("instances " + o2Meta, 1, o2PhysicalOpers.size());
PlanModifier pm = new PlanModifier(plan);
// remove operator w/o removing the stream
pm.removeOperator(o2Meta.getName());
pm.applyChanges(ctx);
Assert.assertEquals("sinks s1 " + s1.getSinks(), 1, s1.getSinks().size());
Assert.assertTrue("undeploy " + ctx.undeploy, ctx.undeploy.containsAll(o2PhysicalOpers));
Set<PTOperator> expDeploy = Sets.newHashSet();
// TODO: container local operators should be included in undeploy/deploy
// expDeploy.addAll(plan.getOperators(o1Meta));
// expDeploy.addAll(plan.getOperators(o3Meta));
Assert.assertEquals("deploy " + ctx.deploy, ctx.deploy, expDeploy);
Assert.assertEquals("streams " + dag.getAllStreams(), 1, dag.getAllStreams().size());
Assert.assertEquals("operators " + dag.getAllOperators(), 2, dag.getAllOperators().size());
Assert.assertTrue("operators " + dag.getAllOperators(), dag.getAllOperators().containsAll(Sets.newHashSet(o1Meta, o3Meta)));
}
use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.
the class PhysicalPlanTest method testStatsListenerContextWrappers.
/**
* Test that internally all stats listeners are handled through StatsListenerWithContext.
* Following cases are tested
*
* Operator implementing StatsListener
* Operator implementing StatsListenerWithContext
* Operator with STATS_LISTENERS attribute set to StatsListener
* Operator with STATS_LISTENERS attribute set to StatsListenerWithContext
*/
@Test
public void testStatsListenerContextWrappers() {
LogicalPlan dag = new LogicalPlan();
dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
StatsListenerOperator o1 = dag.addOperator("o1", new StatsListenerOperator());
GenericTestOperator o2 = dag.addOperator("o2", new GenericTestOperator());
dag.setAttribute(o2, OperatorContext.STATS_LISTENERS, Lists.<StatsListener>newArrayList(mock(StatsListener.class)));
GenericTestOperator o3 = dag.addOperator("o3", new GenericTestOperator());
dag.setAttribute(o3, OperatorContext.STATS_LISTENERS, Lists.<StatsListener>newArrayList(mock(StatsListenerWithContext.class)));
StatsListenerOperatorOld o4 = dag.addOperator("o4", new StatsListenerOperatorOld());
PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
PTOperator p1 = plan.getOperators(dag.getMeta(o1)).get(0);
StatsListener l = p1.statsListeners.get(0);
Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);
PTOperator p2 = plan.getOperators(dag.getMeta(o2)).get(0);
l = p1.statsListeners.get(0);
Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);
PTOperator p3 = plan.getOperators(dag.getMeta(o3)).get(0);
l = p1.statsListeners.get(0);
Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);
PTOperator p4 = plan.getOperators(dag.getMeta(o4)).get(0);
l = p1.statsListeners.get(0);
Assert.assertTrue("Operator stats listener is wrapped ", l instanceof StatsListenerWithContext);
}
use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.
the class PhysicalPlanTest method testContainerCores.
@Test
public void testContainerCores() {
LogicalPlan dag = new LogicalPlan();
dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
GenericTestOperator o4 = dag.addOperator("o4", GenericTestOperator.class);
GenericTestOperator o5 = dag.addOperator("o5", GenericTestOperator.class);
GenericTestOperator o6 = dag.addOperator("o6", GenericTestOperator.class);
dag.setOperatorAttribute(o1, OperatorContext.VCORES, 1);
dag.setOperatorAttribute(o2, OperatorContext.VCORES, 2);
dag.setOperatorAttribute(o3, OperatorContext.VCORES, 3);
dag.setOperatorAttribute(o4, OperatorContext.VCORES, 4);
dag.setOperatorAttribute(o5, OperatorContext.VCORES, 5);
dag.setOperatorAttribute(o6, OperatorContext.VCORES, 6);
dag.addStream("o1.outport1", o1.outport1, o2.inport1).setLocality(Locality.CONTAINER_LOCAL);
dag.addStream("o2.outport1", o2.outport1, o3.inport1, o4.inport1).setLocality(Locality.THREAD_LOCAL);
dag.addStream("o3.output1", o3.outport1, o5.inport1).setLocality(Locality.THREAD_LOCAL);
dag.addStream("o4.output1", o4.outport1, o5.inport2).setLocality(Locality.THREAD_LOCAL);
dag.addStream("o5.output1", o5.outport1, o6.inport1).setLocality(Locality.CONTAINER_LOCAL);
dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
Assert.assertEquals("number of containers", 1, plan.getContainers().size());
Assert.assertEquals("vcores container 1 is 12", 12, plan.getContainers().get(0).getRequiredVCores());
}
Aggregations