Examples with PTOperator com.datatorrent.stram.plan.physical.PTOperator used on opensource projects

Example 61 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class StreamingContainerManager method startRecording.

public void startRecording(String id, int operId, String portName, long numWindows) {
    StreamingContainerAgent sca = getContainerAgentFromOperatorId(operId);
    StramToNodeStartRecordingRequest request = new StramToNodeStartRecordingRequest();
    request.setOperatorId(operId);
    if (!StringUtils.isBlank(portName)) {
        request.setPortName(portName);
    }
    request.setNumWindows(numWindows);
    request.setId(id);
    sca.addOperatorRequest(request);
    PTOperator operator = plan.getAllOperators().get(operId);
    if (operator != null) {
        // restart on deploy
        updateOnDeployRequests(operator, new RecordingRequestFilter(), request);
    }
}

Example 62 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class StreamingContainerManager method deployAfterRestart.

public void deployAfterRestart() {
    if (startedFromCheckpoint) {
        try {
            this.deployChangeInProgress.set(true);
            for (PTContainer c : getPhysicalPlan().getContainers()) {
                c.setState(PTContainer.State.NEW);
                requestContainer(c);
                for (PTOperator ptOperator : c.getOperators()) {
                    ptOperator.setState(State.PENDING_DEPLOY);
                }
            }
        } finally {
            this.deployChangeCnt++;
            this.deployChangeInProgress.set(false);
        }
    }
}

Example 63 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class StreamingContainerManager method findCriticalPath.

private CriticalPathInfo findCriticalPath() {
    CriticalPathInfo result = null;
    List<PTOperator> leafOperators = plan.getLeafOperators();
    Map<PTOperator, CriticalPathInfo> cache = new HashMap<>();
    for (PTOperator leafOperator : leafOperators) {
        CriticalPathInfo cpi = findCriticalPathHelper(leafOperator, cache);
        if (result == null || result.latency < cpi.latency) {
            result = cpi;
        }
    }
    return result;
}

Example 64 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class LogicalPlanModificationTest method testExecutionManager.

private void testExecutionManager(StorageAgent agent) throws Exception {
    dag.setAttribute(OperatorContext.STORAGE_AGENT, agent);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    Assert.assertEquals("" + dnm.containerStartRequests, dnm.containerStartRequests.size(), 0);
    CreateOperatorRequest cor = new CreateOperatorRequest();
    cor.setOperatorFQCN(TestGeneratorInputOperator.class.getName());
    cor.setOperatorName("o1");
    FutureTask<?> lpmf = dnm.logicalPlanModification(Collections.<LogicalPlanRequest>singletonList(cor));
    while (!lpmf.isDone()) {
        dnm.processEvents();
    }
    lpmf.get();
    Assert.assertEquals("" + dnm.containerStartRequests, 1, dnm.containerStartRequests.size());
    PTContainer c = dnm.containerStartRequests.poll().container;
    Assert.assertEquals("operators " + c, 1, c.getOperators().size());
    int deployStatusCnt = 0;
    for (PTOperator oper : c.getOperators()) {
        if (oper.getState() == PTOperator.State.PENDING_DEPLOY) {
            deployStatusCnt++;
        }
    }
    Assert.assertEquals("deploy requests " + c, 1, deployStatusCnt);
    PTOperator oper = c.getOperators().get(0);
    Assert.assertEquals("operator name", "o1", oper.getOperatorMeta().getName());
    Assert.assertEquals("operator class", TestGeneratorInputOperator.class, oper.getOperatorMeta().getOperator().getClass());
}

Example 65 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class LogicalPlanModificationTest method testAddOperatorWithAffinityRules.

@Test
public void testAddOperatorWithAffinityRules() {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.addStream("o2.outport1", o2.outport1, o3.inport1);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    ctx.deploy.clear();
    ctx.undeploy.clear();
    Assert.assertEquals("containers", 3, plan.getContainers().size());
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    rules.add(new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "o1", "added1"));
    rules.add(new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "o3", "added1"));
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    PlanModifier pm = new PlanModifier(plan);
    GenericTestOperator added1 = new GenericTestOperator();
    pm.addOperator("added1", added1);
    pm.addStream("added1.outport1", added1.outport1, o3.inport2);
    Assert.assertEquals("undeploy " + ctx.undeploy, 0, ctx.undeploy.size());
    Assert.assertEquals("deploy " + ctx.deploy, 0, ctx.deploy.size());
    pm.applyChanges(ctx);
    Assert.assertEquals("containers post change", 4, plan.getContainers().size());
    Assert.assertEquals("undeploy " + ctx.undeploy, 1, ctx.undeploy.size());
    Assert.assertEquals("deploy " + ctx.deploy, 2, ctx.deploy.size());
    // Validate affinity rules are applied
    for (PTContainer c : plan.getContainers()) {
        if (c.getOperators().contains("added1")) {
            Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", 2, c.getOperators().size());
            Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", "o1", c.getOperators().get(0).getOperatorMeta().getName());
            Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", "added1", c.getOperators().get(1).getOperatorMeta().getName());
            Set<PTContainer> antiAffinityList = c.getStrictAntiPrefs();
            Assert.assertEquals("There should be one container in antiaffinity list", 1, antiAffinityList.size());
            List<PTOperator> antiAffinityOperators = antiAffinityList.iterator().next().getOperators();
            Assert.assertEquals("AntiAffinity operators should containn operator O3", antiAffinityOperators.iterator().next().getOperatorMeta().getName(), "o3");
        }
    }
}