Examples with PhysicalPlan com.datatorrent.stram.plan.physical.PhysicalPlan used on opensource projects

Example 46 with PhysicalPlan

use of com.datatorrent.stram.plan.physical.PhysicalPlan in project apex-core by apache.

the class StreamCodecTest method testPartitioningMultipleInputStreamCodec.

@Test
public void testPartitioningMultipleInputStreamCodec() {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    dag.setOperatorAttribute(node1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    TestStreamCodec serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe);
    dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 4, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
    LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
    LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);
    for (PTContainer container : containers) {
        List<PTOperator> operators = container.getOperators();
        for (PTOperator operator : operators) {
            if (!operator.isUnifier()) {
                if (operator.getOperatorMeta() == n1meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n1meta.getName(), dnm);
                    OperatorDeployInfo.OutputDeployInfo otdi = getOutputDeployInfo(odi, n1meta.getMeta(node1.outport1));
                    String id = n1meta.getName() + " " + otdi.portName;
                    Assert.assertEquals("number stream codecs " + id, otdi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(n2meta, node2.inport1, otdi.streamCodecs, id, plan);
                } else if (operator.getOperatorMeta() == n2meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n2meta.getName(), dnm);
                    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, n2meta.getMeta(node2.inport1));
                    String id = n2meta.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(n2meta, node2.inport1, idi.streamCodecs, id, plan);
                } else if (operator.getOperatorMeta() == n3meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n3meta.getName(), dnm);
                    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, n3meta.getMeta(node3.inport1));
                    String id = n3meta.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(n3meta, node3.inport1, idi.streamCodecs, id, plan);
                }
            } else {
                OperatorDeployInfo odi = getOperatorDeployInfo(operator, operator.getName(), dnm);
                List<OperatorDeployInfo.InputDeployInfo> idis = odi.inputs;
                for (OperatorDeployInfo.InputDeployInfo idi : idis) {
                    String id = operator.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(n2meta, node2.inport1, idi.streamCodecs, id, plan);
                    checkPresentStreamCodec(n3meta, node3.inport1, idi.streamCodecs, id, plan);
                }
                List<OperatorDeployInfo.OutputDeployInfo> otdis = odi.outputs;
                for (OperatorDeployInfo.OutputDeployInfo otdi : otdis) {
                    String id = operator.getName() + " " + otdi.portName;
                    Assert.assertEquals("number stream codecs " + id, otdi.streamCodecs.size(), 0);
                }
            }
        }
    }
}

Example 47 with PhysicalPlan

use of com.datatorrent.stram.plan.physical.PhysicalPlan in project apex-core by apache.

the class StreamCodecTest method testMultipleStreamCodecs.

@Test
public void testMultipleStreamCodecs() {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    TestStreamCodec serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
    TestStreamCodec2 serDe2 = new TestStreamCodec2();
    dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe2);
    dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 3, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
    LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
    LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);
    OperatorDeployInfo n1di = getSingleOperatorDeployInfo(node1, dnm);
    OperatorDeployInfo.OutputDeployInfo n1odi = getOutputDeployInfo(n1di, n1meta.getMeta(node1.outport1));
    String id = n1meta.getName() + " " + n1odi.portName;
    Assert.assertEquals("number stream codecs " + id, n1odi.streamCodecs.size(), 2);
    checkPresentStreamCodec(n2meta, node2.inport1, n1odi.streamCodecs, id, plan);
    checkPresentStreamCodec(n3meta, node3.inport1, n1odi.streamCodecs, id, plan);
    OperatorDeployInfo n2di = getSingleOperatorDeployInfo(node2, dnm);
    OperatorDeployInfo.InputDeployInfo n2idi = getInputDeployInfo(n2di, n2meta.getMeta(node2.inport1));
    id = n2meta.getName() + " " + n2idi.portName;
    Assert.assertEquals("number stream codecs " + id, n2idi.streamCodecs.size(), 1);
    checkPresentStreamCodec(n2meta, node2.inport1, n2idi.streamCodecs, id, plan);
    OperatorDeployInfo n3di = getSingleOperatorDeployInfo(node3, dnm);
    OperatorDeployInfo.InputDeployInfo n3idi = getInputDeployInfo(n3di, n3meta.getMeta(node3.inport1));
    id = n3meta.getName() + " " + n3idi.portName;
    Assert.assertEquals("number stream codecs " + id, n3idi.streamCodecs.size(), 1);
    checkPresentStreamCodec(n3meta, node3.inport1, n3idi.streamCodecs, id, plan);
}

Example 48 with PhysicalPlan

use of com.datatorrent.stram.plan.physical.PhysicalPlan in project apex-core by apache.

the class CheckpointTest method testUpdateRecoveryCheckpoint.

@Test
public void testUpdateRecoveryCheckpoint() throws Exception {
    Clock clock = new SystemClock();
    dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3SL = dag.addOperator("o3SL", StatelessOperator.class);
    dag.addStream("o1.output1", o1.outport1, o2.inport1);
    dag.addStream("o2.output1", o2.outport1, o3SL.inport1);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    for (PTOperator oper : plan.getAllOperators().values()) {
        Assert.assertEquals("activation windowId " + oper, Checkpoint.INITIAL_CHECKPOINT, oper.getRecoveryCheckpoint());
        Assert.assertEquals("checkpoints " + oper, Collections.emptyList(), oper.checkpoints);
    }
    List<PTOperator> nodes1 = plan.getOperators(dag.getMeta(o1));
    Assert.assertNotNull(nodes1);
    Assert.assertEquals(1, nodes1.size());
    PTOperator o1p1 = nodes1.get(0);
    PTOperator o2p1 = plan.getOperators(dag.getMeta(o2)).get(0);
    PTOperator o3SLp1 = plan.getOperators(dag.getMeta(o3SL)).get(0);
    // recovery checkpoint won't update in deploy state
    for (PTOperator oper : plan.getAllOperators().values()) {
        Assert.assertEquals("", PTOperator.State.PENDING_DEPLOY, oper.getState());
    }
    dnm.updateRecoveryCheckpoints(o2p1, new UpdateCheckpointsContext(clock), false);
    Assert.assertEquals("no checkpoints " + o2p1, Checkpoint.INITIAL_CHECKPOINT, o2p1.getRecoveryCheckpoint());
    UpdateCheckpointsContext ctx = new UpdateCheckpointsContext(clock);
    dnm.updateRecoveryCheckpoints(o1p1, ctx, false);
    Assert.assertEquals("no checkpoints " + o1p1, Checkpoint.INITIAL_CHECKPOINT, o1p1.getRecoveryCheckpoint());
    Assert.assertEquals("number dependencies " + ctx.visited, 3, ctx.visited.size());
    // adding checkpoints to upstream only does not move recovery checkpoint
    Checkpoint cp3 = new Checkpoint(3L, 0, 0);
    Checkpoint cp5 = new Checkpoint(5L, 0, 0);
    Checkpoint cp4 = new Checkpoint(4L, 0, 0);
    o1p1.checkpoints.add(cp3);
    o1p1.checkpoints.add(cp5);
    dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock), false);
    Assert.assertEquals("checkpoint " + o1p1, Checkpoint.INITIAL_CHECKPOINT, o1p1.getRecoveryCheckpoint());
    o2p1.checkpoints.add(new Checkpoint(3L, 0, 0));
    dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock), false);
    Assert.assertEquals("checkpoint " + o1p1, Checkpoint.INITIAL_CHECKPOINT, o1p1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + o2p1, Checkpoint.INITIAL_CHECKPOINT, o2p1.getRecoveryCheckpoint());
    // set leaf operator checkpoint
    dnm.addCheckpoint(o3SLp1, cp5);
    dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock), false);
    Assert.assertEquals("checkpoint " + o1p1, Checkpoint.INITIAL_CHECKPOINT, o1p1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + o2p1, Checkpoint.INITIAL_CHECKPOINT, o2p1.getRecoveryCheckpoint());
    // set all operators as active to enable recovery window id update
    for (PTOperator oper : plan.getAllOperators().values()) {
        oper.setState(PTOperator.State.ACTIVE);
    }
    dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock), false);
    Assert.assertEquals("checkpoint " + o1p1, cp3, o1p1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + o2p1, cp3, o1p1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + o3SLp1, cp5, o3SLp1.getRecoveryCheckpoint());
    Assert.assertNull("checkpoint null for stateless operator " + o3SLp1, o3SLp1.stats.checkpointStats);
    o2p1.checkpoints.add(cp4);
    dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock), false);
    Assert.assertEquals("checkpoint " + o1p1, cp3, o1p1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + o2p1, cp4, o2p1.getRecoveryCheckpoint());
    o1p1.checkpoints.add(1, cp4);
    Assert.assertEquals(o1p1.checkpoints, getCheckpoints(3L, 4L, 5L));
    dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock), false);
    Assert.assertEquals("checkpoint " + o1p1, cp4, o1p1.getRecoveryCheckpoint());
    Assert.assertEquals(o1p1.checkpoints, getCheckpoints(4L, 5L));
    // out of sequence windowIds should be sorted
    dnm.addCheckpoint(o2p1, new Checkpoint(2L, 0, 0));
    Assert.assertEquals("add first", getCheckpoints(2L, 4L), o2p1.checkpoints);
    dnm.addCheckpoint(o2p1, new Checkpoint(3L, 0, 0));
    Assert.assertEquals("add middle", getCheckpoints(2L, 3L, 4L), o2p1.checkpoints);
    dnm.addCheckpoint(o2p1, new Checkpoint(4L, 0, 0));
    Assert.assertEquals("ignore duplicate", getCheckpoints(2L, 3L, 4L), o2p1.checkpoints);
    dnm.addCheckpoint(o2p1, new Checkpoint(5L, 0, 0));
    Assert.assertEquals("add latest", getCheckpoints(2L, 3L, 4L, 5L), o2p1.checkpoints);
}

Example 49 with PhysicalPlan

use of com.datatorrent.stram.plan.physical.PhysicalPlan in project apex-core by apache.

the class CheckpointTest method testUpdateCheckpointsProcessingTimeout.

@Test
public void testUpdateCheckpointsProcessingTimeout() {
    MockClock clock = new MockClock();
    dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    // set all operators as active to enable recovery window id update
    for (PTOperator oper : plan.getAllOperators().values()) {
        oper.setState(PTOperator.State.ACTIVE);
    }
    List<PTOperator> partitions = plan.getOperators(dag.getMeta(o1));
    Assert.assertNotNull(partitions);
    Assert.assertEquals(1, partitions.size());
    PTOperator o1p1 = partitions.get(0);
    partitions = plan.getOperators(dag.getMeta(o2));
    Assert.assertNotNull(partitions);
    Assert.assertEquals(1, partitions.size());
    PTOperator o2p1 = partitions.get(0);
    UpdateCheckpointsContext ctx = new UpdateCheckpointsContext(clock);
    dnm.updateRecoveryCheckpoints(o1p1, ctx, false);
    Assert.assertTrue("no blocked operators", ctx.blocked.isEmpty());
    o1p1.stats.statsRevs.checkout();
    o1p1.stats.currentWindowId.set(1);
    o1p1.stats.lastWindowIdChangeTms = 1;
    o1p1.stats.statsRevs.commit();
    clock.time = o1p1.stats.windowProcessingTimeoutMillis + 1;
    ctx = new UpdateCheckpointsContext(clock);
    dnm.updateRecoveryCheckpoints(o1p1, ctx, false);
    Assert.assertEquals("o2 blocked", Sets.newHashSet(o2p1), ctx.blocked);
    // assign future activation window (state-less or at-most-once).
    Checkpoint cp2 = o2p1.getRecoveryCheckpoint();
    o2p1.setRecoveryCheckpoint(new Checkpoint(o1p1.getRecoveryCheckpoint().windowId + 1, cp2.applicationWindowCount, cp2.checkpointWindowCount));
    ctx = new UpdateCheckpointsContext(clock);
    dnm.updateRecoveryCheckpoints(o1p1, ctx, false);
    Assert.assertEquals("no operators blocked (o2 activation window ahead)", Sets.newHashSet(), ctx.blocked);
    // reset to blocked
    o2p1.setRecoveryCheckpoint(cp2);
    ctx = new UpdateCheckpointsContext(clock);
    dnm.updateRecoveryCheckpoints(o1p1, ctx, false);
    Assert.assertEquals("o2 blocked", Sets.newHashSet(o2p1), ctx.blocked);
    clock.time++;
    ctx = new UpdateCheckpointsContext(clock);
    dnm.updateRecoveryCheckpoints(o1p1, ctx, false);
    Assert.assertEquals("operators blocked", Sets.newHashSet(o1p1, o2p1), ctx.blocked);
    o2p1.stats.statsRevs.checkout();
    o2p1.stats.currentWindowId.set(o1p1.stats.getCurrentWindowId());
    o2p1.stats.statsRevs.commit();
    ctx = new UpdateCheckpointsContext(clock);
    dnm.updateRecoveryCheckpoints(o1p1, ctx, false);
    Assert.assertEquals("operators blocked", Sets.newHashSet(o1p1), ctx.blocked);
    clock.time--;
    ctx = new UpdateCheckpointsContext(clock);
    dnm.updateRecoveryCheckpoints(o1p1, ctx, false);
    Assert.assertEquals("operators blocked", Sets.newHashSet(), ctx.blocked);
}

Example 50 with PhysicalPlan

use of com.datatorrent.stram.plan.physical.PhysicalPlan in project apex-core by apache.

the class CheckpointTest method testUpdateRecoveryCheckpointWithCycle.

@Test
public void testUpdateRecoveryCheckpointWithCycle() throws Exception {
    Clock clock = new SystemClock();
    dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    // Simulate a DAG with a loop which has a unifier operator
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    GenericTestOperator o4 = dag.addOperator("o4", GenericTestOperator.class);
    DefaultDelayOperator d = dag.addOperator("d", DefaultDelayOperator.class);
    dag.addStream("o1.output1", o1.outport, o2.inport1);
    dag.addStream("o2.output1", o2.outport1, o3.inport1);
    dag.addStream("o3.output1", o3.outport1, o4.inport1);
    dag.addStream("o4.output1", o4.outport1, d.input);
    dag.addStream("d.output", d.output, o2.inport2);
    dag.setOperatorAttribute(o3, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<Operator>(2));
    dag.validate();
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    for (PTOperator oper : plan.getAllOperators().values()) {
        Assert.assertEquals("Initial activation windowId" + oper, Checkpoint.INITIAL_CHECKPOINT, oper.getRecoveryCheckpoint());
        Assert.assertEquals("Checkpoints empty" + oper, Collections.emptyList(), oper.checkpoints);
    }
    Checkpoint cp1 = new Checkpoint(1L, 0, 0);
    Checkpoint cp2 = new Checkpoint(2L, 0, 0);
    Map<OperatorMeta, Set<OperatorMeta>> checkpointGroups = dnm.getCheckpointGroups();
    Map<Integer, PTOperator> allOperators = plan.getAllOperators();
    for (PTOperator operator : allOperators.values()) {
        operator.setState(PTOperator.State.ACTIVE);
        operator.checkpoints.add(cp1);
        dnm.updateRecoveryCheckpoints(operator, new UpdateCheckpointsContext(clock, false, checkpointGroups), false);
    }
    List<PTOperator> physicalO1 = plan.getOperators(dag.getOperatorMeta("o1"));
    physicalO1.get(0).checkpoints.add(cp2);
    dnm.updateRecoveryCheckpoints(physicalO1.get(0), new UpdateCheckpointsContext(clock, false, checkpointGroups), false);
    Assert.assertEquals("Recovery checkpoint updated ", physicalO1.get(0).getRecoveryCheckpoint(), cp1);
}