Examples with PTOperator com.datatorrent.stram.plan.physical.PTOperator used on opensource projects

Example 16 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class LogicalPlanModificationTest method testRemoveOperator2.

@Test
public void testRemoveOperator2() {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    OperatorMeta o1Meta = dag.getMeta(o1);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    OperatorMeta o2Meta = dag.getMeta(o2);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    OperatorMeta o3Meta = dag.getMeta(o3);
    LogicalPlan.StreamMeta s1 = dag.addStream("o1.outport1", o1.outport1, o2.inport1, o3.inport1).setLocality(Locality.CONTAINER_LOCAL);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    ctx.deploy.clear();
    ctx.undeploy.clear();
    Assert.assertEquals("containers " + plan.getContainers(), 1, plan.getContainers().size());
    Assert.assertEquals("physical operators " + plan.getAllOperators(), 3, plan.getAllOperators().size());
    Assert.assertEquals("sinks s1 " + s1.getSinks(), 2, s1.getSinks().size());
    List<PTOperator> o2PhysicalOpers = plan.getOperators(o2Meta);
    Assert.assertEquals("instances " + o2Meta, 1, o2PhysicalOpers.size());
    PlanModifier pm = new PlanModifier(plan);
    // remove operator w/o removing the stream
    pm.removeOperator(o2Meta.getName());
    pm.applyChanges(ctx);
    Assert.assertEquals("sinks s1 " + s1.getSinks(), 1, s1.getSinks().size());
    Assert.assertTrue("undeploy " + ctx.undeploy, ctx.undeploy.containsAll(o2PhysicalOpers));
    Set<PTOperator> expDeploy = Sets.newHashSet();
    // TODO: container local operators should be included in undeploy/deploy
    //expDeploy.addAll(plan.getOperators(o1Meta));
    //expDeploy.addAll(plan.getOperators(o3Meta));
    Assert.assertEquals("deploy " + ctx.deploy, ctx.deploy, expDeploy);
    Assert.assertEquals("streams " + dag.getAllStreams(), 1, dag.getAllStreams().size());
    Assert.assertEquals("operators " + dag.getAllOperators(), 2, dag.getAllOperators().size());
    Assert.assertTrue("operators " + dag.getAllOperators(), dag.getAllOperators().containsAll(Sets.newHashSet(o1Meta, o3Meta)));
}

Example 17 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class MockContainer method deploy.

public void deploy() {
    Assert.assertNotNull(sca.container.getExternalId());
    Assert.assertEquals(PTContainer.State.ACTIVE, container.getState());
    //Assert.assertEquals(PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    ContainerStats cstats = new ContainerStats(sca.container.getExternalId());
    ContainerHeartbeat hb = new ContainerHeartbeat();
    hb.setContainerStats(cstats);
    // get deploy request
    ContainerHeartbeatResponse chr = sca.dnmgr.processHeartbeat(hb);
    Assert.assertNotNull(chr.deployRequest);
    Assert.assertEquals("" + chr.deployRequest, container.getOperators().size(), chr.deployRequest.size());
    Assert.assertEquals(PTContainer.State.ACTIVE, container.getState());
    for (PTOperator oper : container.getOperators()) {
        Assert.assertEquals("state " + oper, PTOperator.State.PENDING_DEPLOY, oper.getState());
    }
}

Example 18 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class StreamingContainerManager method findCriticalPathHelper.

private CriticalPathInfo findCriticalPathHelper(PTOperator operator, Map<PTOperator, CriticalPathInfo> cache) {
    CriticalPathInfo cpi = cache.get(operator);
    if (cpi != null) {
        return cpi;
    }
    PTOperator slowestUpstreamOperator = slowestUpstreamOp.get(operator);
    if (slowestUpstreamOperator != null) {
        cpi = findCriticalPathHelper(slowestUpstreamOperator, cache);
        try {
            cpi = (CriticalPathInfo) cpi.clone();
        } catch (CloneNotSupportedException ex) {
            throw new RuntimeException();
        }
    } else {
        cpi = new CriticalPathInfo();
    }
    cpi.latency += operator.stats.getLatencyMA();
    cpi.path.addLast(operator.getId());
    cache.put(operator, cpi);
    return cpi;
}

Example 19 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class StreamingContainerManager method groupByContainer.

private Map<PTContainer, List<PTOperator>> groupByContainer(Collection<PTOperator> operators) {
    Map<PTContainer, List<PTOperator>> m = new HashMap<>();
    for (PTOperator node : operators) {
        List<PTOperator> nodes = m.get(node.getContainer());
        if (nodes == null) {
            nodes = new ArrayList<>();
            m.put(node.getContainer(), nodes);
        }
        nodes.add(node);
    }
    return m;
}

Example 20 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class StreamingContainerManager method getStreamInfoList.

public List<StreamInfo> getStreamInfoList() {
    List<StreamInfo> infoList = new ArrayList<>();
    for (PTContainer container : this.plan.getContainers()) {
        for (PTOperator operator : container.getOperators()) {
            List<PTOutput> outputs = operator.getOutputs();
            for (PTOutput output : outputs) {
                StreamInfo si = new StreamInfo();
                si.logicalName = output.logicalStream.getName();
                si.source.operatorId = String.valueOf(operator.getId());
                si.source.portName = output.portName;
                si.locality = output.logicalStream.getLocality();
                for (PTInput input : output.sinks) {
                    StreamInfo.Port p = new StreamInfo.Port();
                    p.operatorId = String.valueOf(input.target.getId());
                    if (input.target.isUnifier()) {
                        p.portName = StreamingContainer.getUnifierInputPortName(input.portName, operator.getId(), output.portName);
                    } else {
                        p.portName = input.portName;
                    }
                    si.sinks.add(p);
                }
                infoList.add(si);
            }
        }
    }
    return infoList;
}