use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.
the class PhysicalPlanTest method testMxNPartitioning.
/**
* MxN partitioning. When source and sink of a stream are partitioned, a
* separate unifier is created container local with each downstream partition.
*/
@Test
public void testMxNPartitioning() {
LogicalPlan dag = new LogicalPlan();
TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
TestPartitioner<TestGeneratorInputOperator> o1Partitioner = new TestPartitioner<>();
o1Partitioner.setPartitionCount(2);
dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, o1Partitioner);
dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) new PartitioningTest.PartitionLoadWatch()));
OperatorMeta o1Meta = dag.getMeta(o1);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<TestGeneratorInputOperator>(3));
dag.setOperatorAttribute(o2, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
OperatorMeta o2Meta = dag.getMeta(o2);
dag.addStream("o1.outport1", o1.outport, o2.inport1);
TestPlanContext ctx = new TestPlanContext();
dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
PhysicalPlan plan = new PhysicalPlan(dag, ctx);
Assert.assertEquals("number of containers", 5, plan.getContainers().size());
List<PTOperator> inputOperators = new ArrayList<>();
for (int i = 0; i < 2; i++) {
PTContainer container = plan.getContainers().get(i);
Assert.assertEquals("number operators " + container, 1, container.getOperators().size());
Assert.assertEquals("operators " + container, o1Meta.getName(), container.getOperators().get(0).getOperatorMeta().getName());
inputOperators.add(container.getOperators().get(0));
}
for (int i = 2; i < 5; i++) {
PTContainer container = plan.getContainers().get(i);
Assert.assertEquals("number operators " + container, 2, container.getOperators().size());
Assert.assertEquals("operators " + container, o2Meta.getName(), container.getOperators().get(0).getOperatorMeta().getName());
Set<String> expectedLogicalNames = Sets.newHashSet(o1Meta.getMeta(o1.outport).getUnifierMeta().getName(), o2Meta.getName());
Map<String, PTOperator> actualOperators = new HashMap<>();
for (PTOperator p : container.getOperators()) {
actualOperators.put(p.getOperatorMeta().getName(), p);
}
Assert.assertEquals("", expectedLogicalNames, actualOperators.keySet());
PTOperator pUnifier = actualOperators.get(o1Meta.getMeta(o1.outport).getUnifierMeta().getName());
Assert.assertNotNull("" + pUnifier, pUnifier.getContainer());
Assert.assertTrue("" + pUnifier, pUnifier.isUnifier());
// input from each upstream partition
Assert.assertEquals("" + pUnifier, 2, pUnifier.getInputs().size());
int numberPartitionKeys = (i == 2) ? 2 : 1;
for (int inputIndex = 0; inputIndex < pUnifier.getInputs().size(); inputIndex++) {
PTInput input = pUnifier.getInputs().get(inputIndex);
Assert.assertEquals("" + pUnifier, "outport", input.source.portName);
Assert.assertEquals("" + pUnifier, inputOperators.get(inputIndex), input.source.source);
Assert.assertEquals("partition keys " + input.partitions, numberPartitionKeys, input.partitions.partitions.size());
}
// output to single downstream partition
Assert.assertEquals("" + pUnifier, 1, pUnifier.getOutputs().size());
Assert.assertTrue("" + actualOperators.get(o2Meta.getName()).getOperatorMeta().getOperator(), actualOperators.get(o2Meta.getName()).getOperatorMeta().getOperator() instanceof GenericTestOperator);
PTOperator p = actualOperators.get(o2Meta.getName());
Assert.assertEquals("partition inputs " + p.getInputs(), 1, p.getInputs().size());
Assert.assertEquals("partition inputs " + p.getInputs(), pUnifier, p.getInputs().get(0).source.source);
Assert.assertEquals("input partition keys " + p.getInputs(), null, p.getInputs().get(0).partitions);
Assert.assertTrue("partitioned unifier container local " + p.getInputs().get(0).source, p.getInputs().get(0).source.isDownStreamInline());
}
// scale down N from 3 to 2 and then from 2 to 1
for (int i = 0; i < 2; i++) {
List<PTOperator> ptos = plan.getOperators(o2Meta);
Set<PTOperator> expUndeploy = Sets.newHashSet(ptos);
for (PTOperator ptOperator : ptos) {
expUndeploy.addAll(ptOperator.upstreamMerge.values());
PartitioningTest.PartitionLoadWatch.put(ptOperator, -1);
plan.onStatusUpdate(ptOperator);
}
ctx.backupRequests = 0;
ctx.events.remove(0).run();
Set<PTOperator> expDeploy = Sets.newHashSet(plan.getOperators(o2Meta));
// Either unifiers for each partition or single unifier for single partition is expected to be deployed
expDeploy.addAll(plan.getMergeOperators(o1Meta));
for (PTOperator ptOperator : plan.getOperators(o2Meta)) {
expDeploy.addAll(ptOperator.upstreamMerge.values());
}
Assert.assertEquals("number of containers", 4 - i, plan.getContainers().size());
Assert.assertEquals("number of operators", 2 - i, plan.getOperators(o2Meta).size());
Assert.assertEquals("undeployed operators " + ctx.undeploy, expUndeploy, ctx.undeploy);
Assert.assertEquals("deployed operators " + ctx.deploy, expDeploy, ctx.deploy);
}
// scale up N from 1 to 2 and then from 2 to 3
for (int i = 0; i < 2; i++) {
List<PTOperator> unChangedOps = new LinkedList<>(plan.getOperators(o2Meta));
PTOperator o2p1 = unChangedOps.remove(0);
Set<PTOperator> expUndeploy = Sets.newHashSet(o2p1);
// Either single unifier for one partition or merged unifiers for each partition is expected to be undeployed
expUndeploy.addAll(plan.getMergeOperators(o1Meta));
expUndeploy.addAll(o2p1.upstreamMerge.values());
List<PTOperator> nOps = new LinkedList<>();
for (Iterator<PTOperator> iterator = unChangedOps.iterator(); iterator.hasNext(); ) {
PTOperator ptOperator = iterator.next();
nOps.addAll(ptOperator.upstreamMerge.values());
}
unChangedOps.addAll(nOps);
PartitioningTest.PartitionLoadWatch.put(o2p1, 1);
plan.onStatusUpdate(o2p1);
Assert.assertEquals("repartition event", 1, ctx.events.size());
ctx.backupRequests = 0;
ctx.events.remove(0).run();
Assert.assertEquals("N partitions after scale up " + o2Meta, 2 + i, plan.getOperators(o2Meta).size());
Assert.assertTrue("no unifiers", plan.getMergeOperators(o1Meta).isEmpty());
for (PTOperator o : plan.getOperators(o2Meta)) {
Assert.assertNotNull(o.container);
PTOperator unifier = o.upstreamMerge.values().iterator().next();
Assert.assertNotNull(unifier.container);
Assert.assertSame("unifier in same container", o.container, unifier.container);
Assert.assertEquals("container operators " + o.container, Sets.newHashSet(o.container.getOperators()), Sets.newHashSet(o, unifier));
}
Set<PTOperator> expDeploy = Sets.newHashSet(plan.getOperators(o2Meta));
for (PTOperator ptOperator : plan.getOperators(o2Meta)) {
expDeploy.addAll(ptOperator.upstreamMerge.values());
}
expDeploy.removeAll(unChangedOps);
Assert.assertEquals("number of containers", 4 + i, plan.getContainers().size());
Assert.assertEquals("undeployed operators" + ctx.undeploy, expUndeploy, ctx.undeploy);
Assert.assertEquals("deployed operators" + ctx.deploy, expDeploy, ctx.deploy);
}
// scale down M to 1
{
Set<PTOperator> expUndeploy = Sets.newHashSet();
Set<PTOperator> expDeploy = Sets.newHashSet();
for (PTOperator o2p : plan.getOperators(o2Meta)) {
expUndeploy.addAll(o2p.upstreamMerge.values());
expUndeploy.add(o2p);
expDeploy.add(o2p);
}
for (PTOperator o1p : plan.getOperators(o1Meta)) {
expUndeploy.add(o1p);
PartitioningTest.PartitionLoadWatch.put(o1p, -1);
plan.onStatusUpdate(o1p);
}
Assert.assertEquals("repartition event", 1, ctx.events.size());
ctx.events.remove(0).run();
Assert.assertEquals("M partitions after scale down " + o1Meta, 1, plan.getOperators(o1Meta).size());
expUndeploy.removeAll(plan.getOperators(o1Meta));
for (PTOperator o2p : plan.getOperators(o2Meta)) {
Assert.assertTrue("merge unifier " + o2p + " " + o2p.upstreamMerge, o2p.upstreamMerge.isEmpty());
}
Assert.assertEquals("undeploy", expUndeploy, ctx.undeploy);
Assert.assertEquals("deploy", expDeploy, ctx.deploy);
}
// scale up M to 2
Assert.assertEquals("M partitions " + o1Meta, 1, plan.getOperators(o1Meta).size());
{
Set<PTOperator> expUndeploy = Sets.newHashSet();
Set<PTOperator> expDeploy = Sets.newHashSet();
for (PTOperator o1p : plan.getOperators(o1Meta)) {
PartitioningTest.PartitionLoadWatch.put(o1p, 1);
plan.onStatusUpdate(o1p);
}
Assert.assertEquals("repartition event", 1, ctx.events.size());
o1Partitioner.extraPartitions.add(new DefaultPartition<>(o1));
ctx.events.remove(0).run();
o1Partitioner.extraPartitions.clear();
List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
List<PTOperator> o2Partitions = plan.getOperators(o2Meta);
Assert.assertEquals("M partitions after scale up " + o1Meta, 2, o1Partitions.size());
// previous partition unchanged
expDeploy.add(o1Partitions.get(1));
for (PTOperator o1p : o1Partitions) {
Assert.assertEquals("outputs " + o1p, 1, o1p.getOutputs().size());
Assert.assertEquals("sinks " + o1p, o2Partitions.size(), o1p.getOutputs().get(0).sinks.size());
}
for (PTOperator o2p : plan.getOperators(o2Meta)) {
expUndeploy.add(o2p);
expDeploy.add(o2p);
Assert.assertEquals("merge unifier " + o2p + " " + o2p.upstreamMerge, 1, o2p.upstreamMerge.size());
expDeploy.addAll(o2p.upstreamMerge.values());
}
Assert.assertEquals("undeploy", expUndeploy, ctx.undeploy);
Assert.assertEquals("deploy", expDeploy, ctx.deploy);
}
}
use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.
the class PhysicalPlanTest method testSingleFinalUnifierMultiInput.
@Test
public void testSingleFinalUnifierMultiInput() {
LogicalPlan dag = new LogicalPlan();
GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
OperatorMeta o1Meta = dag.getMeta(o1);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(4));
dag.setInputPortAttribute(o2.inport1, PortContext.UNIFIER_SINGLE_FINAL, true);
OperatorMeta o2Meta = dag.getMeta(o2);
GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
dag.setOperatorAttribute(o3, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
OperatorMeta o3Meta = dag.getMeta(o3);
dag.addStream("o1o2o3", o1.outport1, o2.inport1, o3.inport1);
dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 12);
TestPlanContext ctx = new TestPlanContext();
dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
PhysicalPlan plan = new PhysicalPlan(dag, ctx);
Assert.assertEquals("number of containers", 10, plan.getContainers().size());
Assert.assertEquals("o1 merge unifiers", 1, plan.getMergeOperators(o1Meta).size());
// Check the merge unifier
{
PTContainer container = plan.getContainers().get(3);
Assert.assertEquals("number of operators " + container, 1, container.getOperators().size());
PTOperator operator = container.getOperators().get(0);
Assert.assertTrue("unifier check " + operator, operator.isUnifier());
Assert.assertEquals("operator meta " + operator, o1Meta.getMeta(o1.outport1).getUnifierMeta(), operator.getOperatorMeta());
}
int numberO2Containers = 0;
int numberO3Containers = 0;
Set<String> expectedNames = Sets.newHashSet(o1Meta.getMeta(o1.outport1).getUnifierMeta().getName(), o3Meta.getName());
for (int i = 4; i < 10; i++) {
PTContainer container = plan.getContainers().get(i);
List<PTOperator> operators = container.getOperators();
Assert.assertTrue("expected operator count " + container, (operators.size() <= 2) && (operators.size() > 0));
if (operators.size() == 1) {
Assert.assertEquals("operator in container " + container, o2Meta, operators.get(0).getOperatorMeta());
++numberO2Containers;
} else if (operators.size() == 2) {
Set<String> names = Sets.newHashSet();
for (PTOperator operator : container.getOperators()) {
names.add(operator.getOperatorMeta().getName());
}
Assert.assertEquals("container operators " + container, expectedNames, names);
++numberO3Containers;
}
}
Assert.assertEquals("number o2 containers", 4, numberO2Containers);
Assert.assertEquals("number o3 containers", 2, numberO3Containers);
}
use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.
the class PhysicalPlanTest method testStaticPartitioning.
@Test
public void testStaticPartitioning() {
LogicalPlan dag = new LogicalPlan();
dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
TestGeneratorInputOperator node0 = dag.addOperator("node0", TestGeneratorInputOperator.class);
GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
PartitioningTestOperator partitioned = dag.addOperator("partitioned", PartitioningTestOperator.class);
partitioned.setPartitionCount(partitioned.partitionKeys.length);
GenericTestOperator singleton1 = dag.addOperator("singleton1", GenericTestOperator.class);
GenericTestOperator singleton2 = dag.addOperator("singleton2", GenericTestOperator.class);
dag.addStream("n0.inport1", node0.outport, node1.inport1);
dag.addStream("n1.outport1", node1.outport1, partitioned.inport1, partitioned.inportWithCodec);
dag.addStream("mergeStream", partitioned.outport1, singleton1.inport1, singleton2.inport1);
dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
OperatorMeta partitionedMeta = dag.getMeta(partitioned);
dag.validate();
PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
Assert.assertEquals("number of containers", 2, plan.getContainers().size());
Assert.assertNotNull("partition map", partitioned.partitions);
Assert.assertEquals("partition map " + partitioned.partitions, 3, partitioned.partitions.size());
List<PTOperator> n2Instances = plan.getOperators(partitionedMeta);
Assert.assertEquals("partition instances " + n2Instances, partitioned.partitionKeys.length, n2Instances.size());
for (int i = 0; i < n2Instances.size(); i++) {
PTOperator po = n2Instances.get(i);
Map<String, PTInput> inputsMap = new HashMap<>();
for (PTInput input : po.getInputs()) {
inputsMap.put(input.portName, input);
Assert.assertEquals("partitions " + input, Sets.newHashSet(partitioned.partitionKeys[i]), input.partitions.partitions);
//Assert.assertEquals("codec " + input.logicalStream, PartitioningTestStreamCodec.class, input.logicalStream.getCodecClass());
}
Assert.assertEquals("number inputs " + inputsMap, Sets.newHashSet(PartitioningTestOperator.IPORT1, PartitioningTestOperator.INPORT_WITH_CODEC), inputsMap.keySet());
}
Collection<PTOperator> unifiers = plan.getMergeOperators(partitionedMeta);
Assert.assertEquals("number unifiers " + partitionedMeta, 0, unifiers.size());
}
use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.
the class PhysicalPlanTest method testSingleFinalUnifierInputOverride.
@Test
public void testSingleFinalUnifierInputOverride() {
LogicalPlan dag = new LogicalPlan();
GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
OperatorMeta o1Meta = dag.getMeta(o1);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
dag.setInputPortAttribute(o2.inport1, PortContext.UNIFIER_SINGLE_FINAL, true);
OperatorMeta o2Meta = dag.getMeta(o2);
dag.addStream("o1.outport1", o1.outport1, o2.inport1);
dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 10);
TestPlanContext ctx = new TestPlanContext();
dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
PhysicalPlan plan = new PhysicalPlan(dag, ctx);
Assert.assertEquals("number of containers", 6, plan.getContainers().size());
Assert.assertEquals("o1 merge unifiers", 1, plan.getMergeOperators(o1Meta).size());
dag.setOutputPortAttribute(o1.outport1, PortContext.UNIFIER_SINGLE_FINAL, false);
ctx = new TestPlanContext();
dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
plan = new PhysicalPlan(dag, ctx);
Assert.assertEquals("number of containers", 6, plan.getContainers().size());
Assert.assertEquals("o1 merge unifiers", 1, plan.getMergeOperators(o1Meta).size());
dag.setOutputPortAttribute(o1.outport1, PortContext.UNIFIER_SINGLE_FINAL, true);
dag.setInputPortAttribute(o2.inport1, PortContext.UNIFIER_SINGLE_FINAL, false);
ctx = new TestPlanContext();
dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
plan = new PhysicalPlan(dag, ctx);
Assert.assertEquals("number of containers", 5, plan.getContainers().size());
Set<String> expectedNames = Sets.newHashSet(o1Meta.getMeta(o1.outport1).getUnifierMeta().getName(), o2Meta.getName());
for (int i = 3; i < 5; ++i) {
PTContainer container = plan.getContainers().get(i);
Assert.assertEquals("o2 container size", 2, container.getOperators().size());
Set<String> names = Sets.newHashSet();
for (PTOperator operator : container.getOperators()) {
names.add(operator.getOperatorMeta().getName());
}
Assert.assertEquals("o2 container operators", expectedNames, names);
}
}
use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.
the class PhysicalPlanTest method testRepartitioningScaleDownSinglePartition.
/**
* Test unifier gets removed when number partitions drops to 1.
*/
@Test
public void testRepartitioningScaleDownSinglePartition() {
LogicalPlan dag = new LogicalPlan();
TestInputOperator<?> o1 = dag.addOperator("o1", TestInputOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
dag.addStream("o1.outport1", o1.output, o2.inport1);
OperatorMeta o1Meta = dag.getMeta(o1);
dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
TestPlanContext ctx = new TestPlanContext();
dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
PhysicalPlan plan = new PhysicalPlan(dag, ctx);
List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
Assert.assertEquals("partitions " + o1Partitions, 2, o1Partitions.size());
PTOperator o1p1 = o1Partitions.get(0);
PTOperator p1Doper = o1p1.getOutputs().get(0).sinks.get(0).target;
Assert.assertSame("", p1Doper.getOperatorMeta(), o1Meta.getMeta(o1.output).getUnifierMeta());
Assert.assertTrue("unifier ", p1Doper.isUnifier());
Assert.assertEquals("Unifiers " + o1Meta, 1, o1p1.getOutputs().get(0).sinks.size());
Collection<PTOperator> o1Unifiers = new ArrayList<>(plan.getOperators(dag.getMeta(o2)).get(0).upstreamMerge.values());
StatsListener l = o1p1.statsListeners.get(0);
Assert.assertTrue("stats handlers " + o1p1.statsListeners, l instanceof PartitioningTest.PartitionLoadWatch);
PartitioningTest.PartitionLoadWatch.put(o1p1, -1);
PartitioningTest.PartitionLoadWatch.put(o1Partitions.get(1), -1);
plan.onStatusUpdate(o1p1);
plan.onStatusUpdate(o1Partitions.get(1));
Assert.assertEquals("partition scaling triggered", 1, ctx.events.size());
ctx.events.remove(0).run();
List<PTOperator> o1NewPartitions = plan.getOperators(o1Meta);
Assert.assertEquals("partitions " + o1NewPartitions, 1, o1NewPartitions.size());
List<PTOperator> o1NewUnifiers = new ArrayList<>(plan.getOperators(dag.getMeta(o2)).get(0).upstreamMerge.values());
Assert.assertEquals("unifiers " + o1Meta, 0, o1NewUnifiers.size());
p1Doper = o1p1.getOutputs().get(0).sinks.get(0).target;
Assert.assertTrue("", p1Doper.getOperatorMeta() == dag.getMeta(o2));
Assert.assertFalse("unifier ", p1Doper.isUnifier());
Assert.assertTrue("removed unifier from deployment " + ctx.undeploy, ctx.undeploy.containsAll(o1Unifiers));
Assert.assertFalse("removed unifier from deployment " + ctx.deploy, ctx.deploy.containsAll(o1Unifiers));
// scale up, ensure unifier is setup at activation checkpoint
setActivationCheckpoint(o1NewPartitions.get(0), 3);
PartitioningTest.PartitionLoadWatch.put(o1NewPartitions.get(0), 1);
plan.onStatusUpdate(o1NewPartitions.get(0));
Assert.assertEquals("partition scaling triggered", 1, ctx.events.size());
ctx.events.remove(0).run();
o1NewUnifiers.addAll(plan.getOperators(dag.getMeta(o2)).get(0).upstreamMerge.values());
Assert.assertEquals("unifiers " + o1Meta, 1, o1NewUnifiers.size());
Assert.assertEquals("unifier activation checkpoint " + o1Meta, 3, o1NewUnifiers.get(0).recoveryCheckpoint.windowId);
}
Aggregations