Examples with OperatorDeployInfo com.datatorrent.stram.api.OperatorDeployInfo used on opensource projects

Example 11 with OperatorDeployInfo

use of com.datatorrent.stram.api.OperatorDeployInfo in project apex-core by apache.

the class StreamCodecTest method testPartitioningMultipleStreamCodecs.

@Test
public void testPartitioningMultipleStreamCodecs() {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    dag.setOperatorAttribute(node1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    TestStreamCodec serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
    TestStreamCodec2 serDe2 = new TestStreamCodec2();
    dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe2);
    dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 4, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
    LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
    LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);
    for (PTContainer container : containers) {
        List<PTOperator> operators = container.getOperators();
        for (PTOperator operator : operators) {
            if (!operator.isUnifier()) {
                if (operator.getOperatorMeta() == n1meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n1meta.getName(), dnm);
                    OperatorDeployInfo.OutputDeployInfo otdi = getOutputDeployInfo(odi, n1meta.getMeta(node1.outport1));
                    String id = n1meta.getName() + " " + otdi.portName;
                    Assert.assertEquals("number stream codecs " + id, otdi.streamCodecs.size(), 2);
                    checkPresentStreamCodec(n2meta, node2.inport1, otdi.streamCodecs, id, plan);
                    checkPresentStreamCodec(n3meta, node3.inport1, otdi.streamCodecs, id, plan);
                } else if (operator.getOperatorMeta() == n2meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n2meta.getName(), dnm);
                    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, n2meta.getMeta(node2.inport1));
                    String id = n2meta.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(n2meta, node2.inport1, idi.streamCodecs, id, plan);
                } else if (operator.getOperatorMeta() == n3meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n3meta.getName(), dnm);
                    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, n3meta.getMeta(node3.inport1));
                    String id = n3meta.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(n3meta, node3.inport1, idi.streamCodecs, id, plan);
                }
            } else {
                OperatorDeployInfo odi = getOperatorDeployInfo(operator, operator.getName(), dnm);
                Assert.assertEquals("unifier outputs " + operator.getName(), 1, operator.getOutputs().size());
                PTOperator.PTOutput out = operator.getOutputs().get(0);
                Assert.assertEquals("unifier sinks " + operator.getName(), 1, out.sinks.size());
                PTOperator.PTInput idInput = out.sinks.get(0);
                LogicalPlan.OperatorMeta idMeta = idInput.target.getOperatorMeta();
                Operator.InputPort<?> idInputPort = null;
                if (idMeta == n2meta) {
                    idInputPort = node2.inport1;
                } else if (idMeta == n3meta) {
                    idInputPort = node3.inport1;
                }
                List<OperatorDeployInfo.InputDeployInfo> idis = odi.inputs;
                for (OperatorDeployInfo.InputDeployInfo idi : idis) {
                    String id = operator.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(idMeta, idInputPort, idi.streamCodecs, id, plan);
                }
            }
        }
    }
}

Example 12 with OperatorDeployInfo

use of com.datatorrent.stram.api.OperatorDeployInfo in project apex-core by apache.

the class StreamCodecTest method testInlineStreamCodec.

@Test
public void testInlineStreamCodec() {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    TestStreamCodec serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
    dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe);
    dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);
    // Relying on container max count for the manager to layout node1 and node3 in the
    // same container in inline fashion and node2 in a separate container
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 2, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
    LogicalPlan.OperatorMeta nonInlineMeta = null;
    for (int i = 0; i < containers.size(); ++i) {
        PTContainer container = containers.get(i);
        List<PTOperator> operators = container.getOperators();
        if (operators.size() == 1) {
            nonInlineMeta = operators.get(0).getOperatorMeta();
            break;
        }
    }
    Assert.assertNotNull("non inline operator meta is null", nonInlineMeta);
    GenericTestOperator nonInlineOperator = null;
    Operator.InputPort<?> niInputPort = null;
    if (nonInlineMeta.getName().equals("node2")) {
        nonInlineOperator = node2;
        niInputPort = node2.inport1;
    } else if (nonInlineMeta.getName().equals("node3")) {
        nonInlineOperator = node3;
        niInputPort = node3.inport1;
    }
    Assert.assertNotNull("non inline operator is null", nonInlineOperator);
    OperatorDeployInfo n1di = getSingleOperatorDeployInfo(node1, dnm);
    OperatorDeployInfo.OutputDeployInfo n1odi = getOutputDeployInfo(n1di, n1meta.getMeta(node1.outport1));
    String id = n1meta.getName() + " " + n1odi.portName;
    Assert.assertEquals("number stream codecs " + id, n1odi.streamCodecs.size(), 1);
    checkPresentStreamCodec(nonInlineMeta, niInputPort, n1odi.streamCodecs, id, plan);
    OperatorDeployInfo odi = getSingleOperatorDeployInfo(nonInlineOperator, dnm);
    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, nonInlineMeta.getMeta(niInputPort));
    id = nonInlineMeta.getName() + " " + idi.portName;
    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
    checkPresentStreamCodec(nonInlineMeta, niInputPort, idi.streamCodecs, id, plan);
/*
    OperatorDeployInfo n3di = getSingleOperatorDeployInfo(node3, node3.getName(), dnm);

    OperatorDeployInfo.InputDeployInfo n3idi = getInputDeployInfo(n3di, n3meta.getMeta(node3.inport1));
    id = n3meta.getName() + " " + n3idi.portName;
    Assert.assertEquals("number stream codecs " + id, n3idi.streamCodecs.size(), 1);
    streamIdentifier.operName = n3meta.getName();
    streamIdentifier.portName = n3meta.getMeta(node3.inport1).getPortName();
    checkStreamCodecInfo(n3idi.streamCodecs, id, streamIdentifier, serDe2);
    */
}

Example 13 with OperatorDeployInfo

use of com.datatorrent.stram.api.OperatorDeployInfo in project apex-core by apache.

the class StreamingContainer method deploy.

private synchronized void deploy(List<OperatorDeployInfo> nodeList) throws Exception {
    /*
     * A little bit of up front sanity check would reduce the percentage of deploy failures later.
     */
    for (OperatorDeployInfo ndi : nodeList) {
        if (nodes.containsKey(ndi.id)) {
            throw new IllegalStateException("Node with id: " + ndi.id + " already present in container " + containerId + "!");
        }
    }
    deployNodes(nodeList);
    HashMap<String, ArrayList<String>> groupedInputStreams = new HashMap<>();
    for (OperatorDeployInfo ndi : nodeList) {
        groupInputStreams(groupedInputStreams, ndi);
    }
    HashMap<String, ComponentContextPair<Stream, StreamContext>> newStreams = deployOutputStreams(nodeList, groupedInputStreams);
    deployInputStreams(nodeList, newStreams);
    for (ComponentContextPair<Stream, StreamContext> pair : newStreams.values()) {
        pair.component.setup(pair.context);
    }
    streams.putAll(newStreams);
    HashMap<Integer, OperatorDeployInfo> operatorMap = new HashMap<>(nodeList.size());
    for (OperatorDeployInfo o : nodeList) {
        operatorMap.put(o.id, o);
    }
    activate(operatorMap, newStreams);
}

Example 14 with OperatorDeployInfo

use of com.datatorrent.stram.api.OperatorDeployInfo in project apex-core by apache.

the class StreamingContainer method deployNodes.

private void deployNodes(List<OperatorDeployInfo> nodeList) throws IOException {
    for (OperatorDeployInfo ndi : nodeList) {
        StorageAgent backupAgent = getValue(OperatorContext.STORAGE_AGENT, ndi);
        assert (backupAgent != null);
        Context parentContext;
        if (ndi instanceof UnifierDeployInfo) {
            OperatorContext unifiedOperatorContext = new OperatorContext(0, ndi.name, ((UnifierDeployInfo) ndi).operatorAttributes, containerContext);
            parentContext = new PortContext(ndi.inputs.get(0).contextAttributes, unifiedOperatorContext);
            massageUnifierDeployInfo(ndi);
        } else {
            parentContext = containerContext;
        }
        OperatorContext ctx = new OperatorContext(ndi.id, ndi.name, ndi.contextAttributes, parentContext);
        ctx.attributes.put(OperatorContext.ACTIVATION_WINDOW_ID, ndi.checkpoint.windowId);
        logger.debug("Restoring operator {} to checkpoint {} stateless={}.", ndi.id, Codec.getStringWindowId(ndi.checkpoint.windowId), ctx.stateless);
        Node<?> node = Node.retrieveNode(backupAgent.load(ndi.id, ctx.stateless ? Stateless.WINDOW_ID : ndi.checkpoint.windowId), ctx, ndi.type);
        node.currentWindowId = ndi.checkpoint.windowId;
        node.applicationWindowCount = ndi.checkpoint.applicationWindowCount;
        node.firstWindowMillis = firstWindowMillis;
        node.windowWidthMillis = windowWidthMillis;
        node.setId(ndi.id);
        nodes.put(ndi.id, node);
        logger.debug("Marking operator {} as deployed.", node);
    }
}

Example 15 with OperatorDeployInfo

use of com.datatorrent.stram.api.OperatorDeployInfo in project apex-core by apache.

the class StreamingContainer method deployOutputStreams.

private HashMap<String, ComponentContextPair<Stream, StreamContext>> deployOutputStreams(List<OperatorDeployInfo> nodeList, HashMap<String, ArrayList<String>> groupedInputStreams) throws Exception {
    HashMap<String, ComponentContextPair<Stream, StreamContext>> newStreams = new HashMap<>();
    /*
     * We proceed to deploy all the output streams. At the end of this block, our streams collection
     * will contain all the streams which originate at the output port of the operators. The streams
     * are generally mapped against the "nodename.portname" string. But the BufferServerPublishers which
     * share the output port with other inline streams are mapped against the Buffer Server port to
     * avoid collision and at the same time keep track of these buffer streams.
     */
    for (OperatorDeployInfo ndi : nodeList) {
        Node<?> node = nodes.get(ndi.id);
        long checkpointWindowId = ndi.checkpoint.windowId;
        for (OperatorDeployInfo.OutputDeployInfo nodi : ndi.outputs) {
            String sourceIdentifier = Integer.toString(ndi.id).concat(Component.CONCAT_SEPARATOR).concat(nodi.portName);
            int queueCapacity = getValue(PortContext.QUEUE_CAPACITY, nodi, ndi);
            logger.debug("for stream {} the queue capacity is {}", sourceIdentifier, queueCapacity);
            ArrayList<String> collection = groupedInputStreams.get(sourceIdentifier);
            Map<Integer, StreamCodec<?>> streamCodecs = nodi.streamCodecs;
            if ((collection == null) && (streamCodecs.size() == 1)) {
                assert (nodi.bufferServerHost != null) : "resulting stream cannot be inline: " + nodi;
                /*
           * Let's create a stream to carry the data to the Buffer Server.
           * Nobody in this container is interested in the output placed on this stream, but
           * this stream exists. That means someone outside of this container must be interested.
           */
                Map.Entry<Integer, StreamCodec<?>> entry = streamCodecs.entrySet().iterator().next();
                StreamCodec<?> streamCodec = entry.getValue();
                Integer streamCodecIdentifier = entry.getKey();
                String connIdentifier = sourceIdentifier + Component.CONCAT_SEPARATOR + streamCodecIdentifier;
                SimpleEntry<String, ComponentContextPair<Stream, StreamContext>> deployBufferServerPublisher = deployBufferServerPublisher(connIdentifier, streamCodec, checkpointWindowId, queueCapacity, nodi);
                newStreams.put(sourceIdentifier, deployBufferServerPublisher.getValue());
                node.connectOutputPort(nodi.portName, deployBufferServerPublisher.getValue().component);
            } else {
                /*
           * In this case we have 2 possibilities, either we have 1 inline or multiple streams.
           * Since we cannot tell at this point, we assume that we will have multiple streams and
           * plan accordingly. we possibly will come to this code block multiple times. We create
           * the MuxStream only the first time and use it for subsequent calls of this block.
           *
           * There is also the possibility that we have a stream with multiple sinks having distinct codecs
           */
                ComponentContextPair<Stream, StreamContext> pair = newStreams.get(sourceIdentifier);
                if (pair == null) {
                    /**
             * Let's multiplex the output placed on this stream.
             * This container itself contains more than one parties interested.
             */
                    StreamContext context = new StreamContext(nodi.declaredStreamId);
                    context.setSourceId(sourceIdentifier);
                    context.setFinishedWindowId(checkpointWindowId);
                    Stream stream = new MuxStream();
                    newStreams.put(sourceIdentifier, pair = new ComponentContextPair<>(stream, context));
                    node.connectOutputPort(nodi.portName, stream);
                }
                if (nodi.bufferServerHost != null) {
                    /*
             * Although there is a node in this container interested in output placed on this stream, there
             * seems to at least one more party interested but placed in a container other than this one.
             */
                    for (Map.Entry<Integer, StreamCodec<?>> entry : streamCodecs.entrySet()) {
                        Integer streamCodecIdentifier = entry.getKey();
                        StreamCodec<?> streamCodec = entry.getValue();
                        String connIdentifier = sourceIdentifier + Component.CONCAT_SEPARATOR + streamCodecIdentifier;
                        SimpleEntry<String, ComponentContextPair<Stream, StreamContext>> deployBufferServerPublisher = deployBufferServerPublisher(connIdentifier, streamCodec, checkpointWindowId, queueCapacity, nodi);
                        newStreams.put(deployBufferServerPublisher.getKey(), deployBufferServerPublisher.getValue());
                        String sinkIdentifier = pair.context.getSinkId();
                        if (sinkIdentifier == null) {
                            pair.context.setSinkId(deployBufferServerPublisher.getKey());
                        } else {
                            pair.context.setSinkId(sinkIdentifier.concat(", ").concat(deployBufferServerPublisher.getKey()));
                        }
                        ((Stream.MultiSinkCapableStream) pair.component).setSink(deployBufferServerPublisher.getKey(), deployBufferServerPublisher.getValue().component);
                    }
                }
            }
        }
    }
    return newStreams;
}