Examples with PTContainer com.datatorrent.stram.plan.physical.PTContainer used on opensource projects

Example 46 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StreamingContainerManager method deployAfterRestart.

public void deployAfterRestart() {
    if (startedFromCheckpoint) {
        try {
            this.deployChangeInProgress.set(true);
            for (PTContainer c : getPhysicalPlan().getContainers()) {
                c.setState(PTContainer.State.NEW);
                requestContainer(c);
                for (PTOperator ptOperator : c.getOperators()) {
                    ptOperator.setState(State.PENDING_DEPLOY);
                }
            }
        } finally {
            this.deployChangeCnt++;
            this.deployChangeInProgress.set(false);
        }
    }
}

Example 47 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class LogicalPlanModificationTest method testExecutionManager.

private void testExecutionManager(StorageAgent agent) throws Exception {
    dag.setAttribute(OperatorContext.STORAGE_AGENT, agent);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    Assert.assertEquals("" + dnm.containerStartRequests, dnm.containerStartRequests.size(), 0);
    CreateOperatorRequest cor = new CreateOperatorRequest();
    cor.setOperatorFQCN(TestGeneratorInputOperator.class.getName());
    cor.setOperatorName("o1");
    FutureTask<?> lpmf = dnm.logicalPlanModification(Collections.<LogicalPlanRequest>singletonList(cor));
    while (!lpmf.isDone()) {
        dnm.processEvents();
    }
    lpmf.get();
    Assert.assertEquals("" + dnm.containerStartRequests, 1, dnm.containerStartRequests.size());
    PTContainer c = dnm.containerStartRequests.poll().container;
    Assert.assertEquals("operators " + c, 1, c.getOperators().size());
    int deployStatusCnt = 0;
    for (PTOperator oper : c.getOperators()) {
        if (oper.getState() == PTOperator.State.PENDING_DEPLOY) {
            deployStatusCnt++;
        }
    }
    Assert.assertEquals("deploy requests " + c, 1, deployStatusCnt);
    PTOperator oper = c.getOperators().get(0);
    Assert.assertEquals("operator name", "o1", oper.getOperatorMeta().getName());
    Assert.assertEquals("operator class", TestGeneratorInputOperator.class, oper.getOperatorMeta().getOperator().getClass());
}

Example 48 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class LogicalPlanModificationTest method testAddOperatorWithAffinityRules.

@Test
public void testAddOperatorWithAffinityRules() {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.addStream("o2.outport1", o2.outport1, o3.inport1);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    ctx.deploy.clear();
    ctx.undeploy.clear();
    Assert.assertEquals("containers", 3, plan.getContainers().size());
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    rules.add(new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "o1", "added1"));
    rules.add(new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "o3", "added1"));
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    PlanModifier pm = new PlanModifier(plan);
    GenericTestOperator added1 = new GenericTestOperator();
    pm.addOperator("added1", added1);
    pm.addStream("added1.outport1", added1.outport1, o3.inport2);
    Assert.assertEquals("undeploy " + ctx.undeploy, 0, ctx.undeploy.size());
    Assert.assertEquals("deploy " + ctx.deploy, 0, ctx.deploy.size());
    pm.applyChanges(ctx);
    Assert.assertEquals("containers post change", 4, plan.getContainers().size());
    Assert.assertEquals("undeploy " + ctx.undeploy, 1, ctx.undeploy.size());
    Assert.assertEquals("deploy " + ctx.deploy, 2, ctx.deploy.size());
    // Validate affinity rules are applied
    for (PTContainer c : plan.getContainers()) {
        if (c.getOperators().contains("added1")) {
            Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", 2, c.getOperators().size());
            Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", "o1", c.getOperators().get(0).getOperatorMeta().getName());
            Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", "added1", c.getOperators().get(1).getOperatorMeta().getName());
            Set<PTContainer> antiAffinityList = c.getStrictAntiPrefs();
            Assert.assertEquals("There should be one container in antiaffinity list", 1, antiAffinityList.size());
            List<PTOperator> antiAffinityOperators = antiAffinityList.iterator().next().getOperators();
            Assert.assertEquals("AntiAffinity operators should containn operator O3", antiAffinityOperators.iterator().next().getOperatorMeta().getName(), "o3");
        }
    }
}

Example 49 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StreamingContainerManager method getHeartbeatResponse.

private ContainerHeartbeatResponse getHeartbeatResponse(StreamingContainerAgent sca) {
    ContainerHeartbeatResponse rsp = new ContainerHeartbeatResponse();
    if (this.deployChangeInProgress.get() || sca.deployCnt != this.deployChangeCnt) {
        LOG.debug("{} deferred requests due to concurrent plan change.", sca.container.toIdStateString());
        rsp.hasPendingRequests = true;
        return rsp;
    }
    if (!sca.undeployOpers.isEmpty()) {
        rsp.undeployRequest = Lists.newArrayList(sca.undeployOpers);
        rsp.hasPendingRequests = (!sca.deployOpers.isEmpty());
        return rsp;
    }
    Set<PTOperator> deployOperators = sca.deployOpers;
    if (!deployOperators.isEmpty()) {
        // deploy once all containers are running and no undeploy operations are pending.
        for (PTContainer c : getPhysicalPlan().getContainers()) {
            if (c.getState() != PTContainer.State.ACTIVE) {
                LOG.debug("{} waiting for container activation {}", sca.container.toIdStateString(), c.toIdStateString());
                rsp.hasPendingRequests = true;
                return rsp;
            }
            for (PTOperator oper : c.getOperators()) {
                if (oper.getState() == PTOperator.State.PENDING_UNDEPLOY) {
                    LOG.debug("{} waiting for undeploy {} {}", sca.container.toIdStateString(), c.toIdStateString(), oper);
                    rsp.hasPendingRequests = true;
                    return rsp;
                }
            }
        }
        LOG.debug("{} deployable operators: {}", sca.container.toIdStateString(), deployOperators);
        List<OperatorDeployInfo> deployList = sca.getDeployInfoList(deployOperators);
        if (deployList != null && !deployList.isEmpty()) {
            rsp.deployRequest = deployList;
            rsp.nodeRequests = Lists.newArrayList();
            for (PTOperator o : deployOperators) {
                rsp.nodeRequests.addAll(o.deployRequests);
            }
        }
        rsp.hasPendingRequests = false;
        return rsp;
    }
    return rsp;
}

Example 50 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StreamingContainerAgent method getDeployInfoList.

/**
   * Create deploy info for StramChild.
   *
   * @param operators
   * @return StreamingContainerContext
   */
public List<OperatorDeployInfo> getDeployInfoList(Collection<PTOperator> operators) {
    if (container.bufferServerAddress == null) {
        throw new AssertionError("No buffer server address assigned");
    }
    Map<OperatorDeployInfo, PTOperator> nodes = new LinkedHashMap<>();
    HashSet<PTOperator.PTOutput> publishers = new HashSet<>();
    PhysicalPlan physicalPlan = dnmgr.getPhysicalPlan();
    for (PTOperator oper : operators) {
        if (oper.getState() != State.PENDING_DEPLOY) {
            LOG.debug("Skipping deploy for operator {} state {}", oper, oper.getState());
            continue;
        }
        OperatorDeployInfo ndi = createOperatorDeployInfo(oper);
        nodes.put(ndi, oper);
        ndi.inputs = new ArrayList<>(oper.getInputs().size());
        ndi.outputs = new ArrayList<>(oper.getOutputs().size());
        for (PTOperator.PTOutput out : oper.getOutputs()) {
            final StreamMeta streamMeta = out.logicalStream;
            // buffer server or inline publisher
            OutputDeployInfo portInfo = new OutputDeployInfo();
            portInfo.declaredStreamId = streamMeta.getName();
            portInfo.portName = out.portName;
            try {
                portInfo.contextAttributes = streamMeta.getSource().getAttributes().clone();
            } catch (CloneNotSupportedException ex) {
                throw new RuntimeException("Cannot clone attributes", ex);
            }
            boolean outputUnified = false;
            for (PTOperator.PTInput input : out.sinks) {
                if (input.target.isUnifier()) {
                    outputUnified = true;
                    break;
                }
            }
            portInfo.contextAttributes.put(PortContext.IS_OUTPUT_UNIFIED, outputUnified);
            if (ndi.type == OperatorDeployInfo.OperatorType.UNIFIER) {
                // input attributes of the downstream operator
                for (InputPortMeta sink : streamMeta.getSinks()) {
                    try {
                        portInfo.contextAttributes = sink.getAttributes().clone();
                    } catch (CloneNotSupportedException e) {
                        throw new RuntimeException("Cannot clone attributes", e);
                    }
                    break;
                }
            }
            if (!out.isDownStreamInline()) {
                portInfo.bufferServerHost = oper.getContainer().bufferServerAddress.getHostName();
                portInfo.bufferServerPort = oper.getContainer().bufferServerAddress.getPort();
                portInfo.bufferServerToken = oper.getContainer().getBufferServerToken();
                // Build the stream codec configuration of all sinks connected to this port
                for (PTOperator.PTInput input : out.sinks) {
                    // Create mappings for all non-inline operators
                    if (input.target.getContainer() != out.source.getContainer()) {
                        final StreamCodec<?> streamCodec = getIdentifyingInputPortMeta(input).getStreamCodec();
                        final Integer id = physicalPlan.getStreamCodecIdentifier(streamCodec);
                        // TODO: replace with inputInfo.streamCodecs.putIfAbsent() after support for JDK 1.7 is dropped.
                        if (!portInfo.streamCodecs.containsKey(id)) {
                            portInfo.streamCodecs.put(id, streamCodec);
                        }
                    }
                }
            }
            ndi.outputs.add(portInfo);
            publishers.add(out);
        }
    }
    for (Map.Entry<OperatorDeployInfo, PTOperator> operEntry : nodes.entrySet()) {
        OperatorDeployInfo ndi = operEntry.getKey();
        PTOperator oper = operEntry.getValue();
        for (PTOperator.PTInput in : oper.getInputs()) {
            final StreamMeta streamMeta = in.logicalStream;
            if (streamMeta.getSource() == null) {
                throw new AssertionError("source is null: " + in);
            }
            PTOperator.PTOutput sourceOutput = in.source;
            InputDeployInfo inputInfo = new InputDeployInfo();
            inputInfo.declaredStreamId = streamMeta.getName();
            inputInfo.portName = in.portName;
            InputPortMeta inputPortMeta = getInputPortMeta(oper.getOperatorMeta(), streamMeta);
            if (inputPortMeta != null) {
                try {
                    inputInfo.contextAttributes = inputPortMeta.getAttributes().clone();
                } catch (CloneNotSupportedException e) {
                    throw new RuntimeException("Cannot clone attributes", e);
                }
            }
            if (inputInfo.contextAttributes == null && ndi.type == OperatorDeployInfo.OperatorType.UNIFIER) {
                try {
                    inputInfo.contextAttributes = in.source.logicalStream.getSource().getAttributes().clone();
                } catch (CloneNotSupportedException e) {
                    throw new RuntimeException("Cannot clone attributes", e);
                }
            }
            inputInfo.sourceNodeId = sourceOutput.source.getId();
            inputInfo.sourcePortName = sourceOutput.portName;
            if (in.partitions != null && in.partitions.mask != 0) {
                inputInfo.partitionMask = in.partitions.mask;
                inputInfo.partitionKeys = in.partitions.partitions;
            }
            if (sourceOutput.source.getContainer() == oper.getContainer()) {
                // both operators in same container
                if (!publishers.contains(sourceOutput)) {
                    throw new AssertionError("Source not deployed for container local stream " + sourceOutput + " " + in);
                }
                if (streamMeta.getLocality() == Locality.THREAD_LOCAL) {
                    inputInfo.locality = Locality.THREAD_LOCAL;
                    ndi.type = OperatorType.OIO;
                } else {
                    inputInfo.locality = Locality.CONTAINER_LOCAL;
                }
            } else {
                // buffer server input
                PTContainer container = sourceOutput.source.getContainer();
                InetSocketAddress addr = container.bufferServerAddress;
                if (addr == null) {
                    throw new AssertionError("upstream address not assigned: " + sourceOutput);
                }
                inputInfo.bufferServerHost = addr.getHostName();
                inputInfo.bufferServerPort = addr.getPort();
                inputInfo.bufferServerToken = container.getBufferServerToken();
            }
            // On the input side there is a unlikely scenario of partitions even for inline stream that is being
            // handled. Always specifying a stream codec configuration in case that scenario happens.
            final StreamCodec<?> streamCodec = getIdentifyingInputPortMeta(in).getStreamCodec();
            final Integer id = physicalPlan.getStreamCodecIdentifier(streamCodec);
            // TODO: replace with inputInfo.streamCodecs.putIfAbsent() after support for JDK 1.7 is dropped.
            if (!inputInfo.streamCodecs.containsKey(id)) {
                inputInfo.streamCodecs.put(id, streamCodec);
            }
            ndi.inputs.add(inputInfo);
        }
    }
    return new ArrayList<>(nodes.keySet());
}