Examples with InputPortMeta com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta used on opensource projects

Example 1 with InputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta in project apex-core by apache.

the class StreamPersistanceTests method testDynamicPartitioning.

@Test
public void testDynamicPartitioning() throws ClassNotFoundException, IOException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    dag.setOperatorAttribute(console, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TestReceiverOperator>(2));
    dag.setOperatorAttribute(console, Context.OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) new PartitioningTest.PartitionLoadWatch()));
    final PartitionedTestPersistanceOperator console1 = new PartitionedTestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, console.inport);
    dag.setInputPortAttribute(console.inport, PortContext.STREAM_CODEC, new TestPartitionCodec());
    s.persistUsing("persister", console1, console1.inport);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 4, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta passThruMeta = dag.getMeta(console);
    List<PTOperator> ptos = plan.getOperators(passThruMeta);
    PTOperator persistOperatorContainer = null;
    for (PTContainer container : plan.getContainers()) {
        for (PTOperator operator : container.getOperators()) {
            operator.setState(PTOperator.State.ACTIVE);
            if (operator.getName().equals("persister")) {
                persistOperatorContainer = operator;
            }
        }
    }
    // Check that persist operator is part of dependents redeployed
    Set<PTOperator> operators = plan.getDependents(ptos);
    logger.debug("Operators to be re-deployed = {}", operators);
    // Validate that persist operator is part of dependents
    assertTrue("persist operator should be part of the operators to be redeployed", operators.contains(persistOperatorContainer));
    LogicalPlan.StreamMeta s1 = (LogicalPlan.StreamMeta) s;
    StreamCodec codec = s1.getPersistOperatorInputPort().getStreamCodec();
    assertEquals("Codec should be instance of StreamCodecWrapper", codec instanceof StreamCodecWrapperForPersistance, true);
    StreamCodecWrapperForPersistance wrapperCodec = (StreamCodecWrapperForPersistance) codec;
    Entry<InputPortMeta, Collection<PartitionKeys>> keys = (Entry<InputPortMeta, Collection<PartitionKeys>>) wrapperCodec.inputPortToPartitionMap.entrySet().iterator().next();
    logger.debug(keys.toString());
    assertEquals("Size of partitions should be 2", 2, keys.getValue().size());
    for (PTOperator ptOperator : ptos) {
        PartitioningTest.PartitionLoadWatch.put(ptOperator, -1);
        plan.onStatusUpdate(ptOperator);
    }
    dnm.processEvents();
    assertEquals("Input port map", wrapperCodec.inputPortToPartitionMap.size(), 1);
    keys = (Entry<InputPortMeta, Collection<PartitionKeys>>) wrapperCodec.inputPortToPartitionMap.entrySet().iterator().next();
    assertEquals("Size of partitions should be 1 after repartition", 1, keys.getValue().size());
    logger.debug(keys.toString());
}

Example 2 with InputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta in project apex-core by apache.

the class StreamingContainerManager method getPortAttributes.

public Attribute.AttributeMap getPortAttributes(String operatorId, String portName) {
    OperatorMeta logicalOperator = plan.getLogicalPlan().getOperatorMeta(operatorId);
    if (logicalOperator == null) {
        throw new IllegalArgumentException("Invalid operatorId " + operatorId);
    }
    Operators.PortMappingDescriptor portMap = new Operators.PortMappingDescriptor();
    Operators.describe(logicalOperator.getOperator(), portMap);
    PortContextPair<InputPort<?>> inputPort = portMap.inputPorts.get(portName);
    if (inputPort != null) {
        InputPortMeta portMeta = logicalOperator.getMeta(inputPort.component);
        try {
            return portMeta.getAttributes().clone();
        } catch (CloneNotSupportedException ex) {
            throw new RuntimeException("Cannot clone port attributes", ex);
        }
    } else {
        PortContextPair<OutputPort<?>> outputPort = portMap.outputPorts.get(portName);
        if (outputPort != null) {
            OutputPortMeta portMeta = logicalOperator.getMeta(outputPort.component);
            try {
                return portMeta.getAttributes().clone();
            } catch (CloneNotSupportedException ex) {
                throw new RuntimeException("Cannot clone port attributes", ex);
            }
        }
        throw new IllegalArgumentException("Invalid port name " + portName);
    }
}

Example 3 with InputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta in project apex-core by apache.

the class LogicalPlanConfiguration method setOperatorConfiguration.

private void setOperatorConfiguration(final LogicalPlan dag, List<AppConf> appConfs, String appName) {
    for (final OperatorMeta ow : dag.getAllOperators()) {
        List<OperatorConf> opConfs = getMatchingChildConf(appConfs, ow.getName(), StramElement.OPERATOR);
        // Set the operator attributes
        setAttributes(opConfs, ow.getAttributes());
        // Set the operator opProps
        Map<String, String> opProps = getProperties(getPropertyArgs(ow), opConfs, appName);
        setOperatorProperties(ow.getOperator(), opProps);
        // Set the port attributes
        for (Entry<LogicalPlan.InputPortMeta, LogicalPlan.StreamMeta> entry : ow.getInputStreams().entrySet()) {
            final InputPortMeta im = entry.getKey();
            List<PortConf> inPortConfs = getMatchingChildConf(opConfs, im.getPortName(), StramElement.INPUT_PORT);
            // Add the generic port attributes as well
            List<PortConf> portConfs = getMatchingChildConf(opConfs, im.getPortName(), StramElement.PORT);
            inPortConfs.addAll(portConfs);
            setAttributes(inPortConfs, im.getAttributes());
        }
        for (Entry<LogicalPlan.OutputPortMeta, LogicalPlan.StreamMeta> entry : ow.getOutputStreams().entrySet()) {
            final OutputPortMeta om = entry.getKey();
            List<PortConf> outPortConfs = getMatchingChildConf(opConfs, om.getPortName(), StramElement.OUTPUT_PORT);
            // Add the generic port attributes as well
            List<PortConf> portConfs = getMatchingChildConf(opConfs, om.getPortName(), StramElement.PORT);
            outPortConfs.addAll(portConfs);
            setAttributes(outPortConfs, om.getAttributes());
            List<OperatorConf> unifConfs = getMatchingChildConf(outPortConfs, null, StramElement.UNIFIER);
            if (unifConfs.size() != 0) {
                setAttributes(unifConfs, om.getUnifierMeta().getAttributes());
            }
        }
        ow.populateAggregatorMeta();
    }
}

Example 4 with InputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta in project apex-core by apache.

the class PhysicalPlan method updatePersistOperatorStreamCodec.

private void updatePersistOperatorStreamCodec(LogicalPlan dag) {
    HashMap<StreamMeta, StreamCodec<?>> streamMetaToCodecMap = new HashMap<>();
    try {
        for (OperatorMeta n : dag.getAllOperators()) {
            for (StreamMeta s : n.getOutputStreams().values()) {
                if (s.getPersistOperator() != null) {
                    Map<InputPortMeta, StreamCodec<?>> inputStreamCodecs = new HashMap<>();
                    // Logging is enabled for the stream
                    for (InputPortMeta portMeta : s.getSinksToPersist()) {
                        StreamCodec<?> inputStreamCodec = portMeta.getStreamCodec();
                        if (inputStreamCodec != null) {
                            boolean alreadyAdded = false;
                            for (StreamCodec<?> codec : inputStreamCodecs.values()) {
                                if (inputStreamCodec.equals(codec)) {
                                    alreadyAdded = true;
                                    break;
                                }
                            }
                            if (!alreadyAdded) {
                                inputStreamCodecs.put(portMeta, inputStreamCodec);
                            }
                        }
                    }
                    if (inputStreamCodecs.isEmpty()) {
                    // Stream codec not specified
                    // So everything out of Source should be captured without any
                    // StreamCodec
                    // Do nothing
                    } else {
                        // Create Wrapper codec for Stream persistence using all unique
                        // stream codecs
                        // Logger should write merged or union of all input stream codecs
                        StreamCodec<?> specifiedCodecForLogger = s.getPersistOperatorInputPort().getStreamCodec();
                        @SuppressWarnings({ "unchecked", "rawtypes" }) StreamCodecWrapperForPersistance<Object> codec = new StreamCodecWrapperForPersistance(inputStreamCodecs, specifiedCodecForLogger);
                        streamMetaToCodecMap.put(s, codec);
                    }
                }
            }
        }
        for (java.util.Map.Entry<StreamMeta, StreamCodec<?>> entry : streamMetaToCodecMap.entrySet()) {
            dag.setInputPortAttribute(entry.getKey().getPersistOperatorInputPort().getPort(), PortContext.STREAM_CODEC, entry.getValue());
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 5 with InputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta in project apex-core by apache.

the class PhysicalPlan method updateStreamMappings.

private void updateStreamMappings(PMapping m) {
    for (Map.Entry<OutputPortMeta, StreamMeta> opm : m.logicalOperator.getOutputStreams().entrySet()) {
        StreamMapping ug = m.outputStreams.get(opm.getKey());
        if (ug == null) {
            ug = new StreamMapping(opm.getValue(), this);
            m.outputStreams.put(opm.getKey(), ug);
        }
        LOG.debug("update stream mapping for {} {}", opm.getKey().getOperatorMeta(), opm.getKey().getPortName());
        ug.setSources(m.partitions);
    }
    for (Map.Entry<InputPortMeta, StreamMeta> ipm : m.logicalOperator.getInputStreams().entrySet()) {
        PMapping sourceMapping = this.logicalToPTOperator.get(ipm.getValue().getSource().getOperatorMeta());
        if (ipm.getValue().getSource().getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
            // skip if the source is a DelayOperator
            continue;
        }
        if (ipm.getKey().getValue(PortContext.PARTITION_PARALLEL)) {
            if (sourceMapping.partitions.size() < m.partitions.size()) {
                throw new AssertionError("Number of partitions don't match in parallel mapping " + sourceMapping.logicalOperator.getName() + " -> " + m.logicalOperator.getName() + ", " + sourceMapping.partitions.size() + " -> " + m.partitions.size());
            }
            int slidingWindowCount = 0;
            OperatorMeta sourceOM = sourceMapping.logicalOperator;
            if (sourceOM.getAttributes().contains(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT)) {
                if (sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT) < sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT)) {
                    slidingWindowCount = sourceOM.getValue(OperatorContext.SLIDE_BY_WINDOW_COUNT);
                } else {
                    LOG.warn("Sliding Window Count {} should be less than APPLICATION WINDOW COUNT {}", sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT), sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT));
                }
            }
            for (int i = 0; i < m.partitions.size(); i++) {
                PTOperator oper = m.partitions.get(i);
                PTOperator sourceOper = sourceMapping.partitions.get(i);
                for (PTOutput sourceOut : sourceOper.outputs) {
                    nextSource: if (sourceOut.logicalStream == ipm.getValue()) {
                        //avoid duplicate entries in case of parallel partitions
                        for (PTInput sinkIn : sourceOut.sinks) {
                            // input-port-meta currently being looked at since we allow an output port to connect to multiple inputs of the same operator.
                            if (sinkIn.target == oper && sinkIn.portName.equals(ipm.getKey().getPortName())) {
                                break nextSource;
                            }
                        }
                        PTInput input;
                        if (slidingWindowCount > 0) {
                            PTOperator slidingUnifier = StreamMapping.createSlidingUnifier(sourceOut.logicalStream, this, sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT), slidingWindowCount);
                            StreamMapping.addInput(slidingUnifier, sourceOut, null);
                            input = new PTInput(ipm.getKey().getPortName(), ipm.getValue(), oper, null, slidingUnifier.outputs.get(0), ipm.getKey().getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR));
                            sourceMapping.outputStreams.get(ipm.getValue().getSource()).slidingUnifiers.add(slidingUnifier);
                        } else {
                            input = new PTInput(ipm.getKey().getPortName(), ipm.getValue(), oper, null, sourceOut, ipm.getKey().getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR));
                        }
                        oper.inputs.add(input);
                    }
                }
            }
        } else {
            StreamMapping ug = sourceMapping.outputStreams.get(ipm.getValue().getSource());
            if (ug == null) {
                ug = new StreamMapping(ipm.getValue(), this);
                m.outputStreams.put(ipm.getValue().getSource(), ug);
            }
            LOG.debug("update upstream stream mapping for {} {}", sourceMapping.logicalOperator, ipm.getValue().getSource().getPortName());
            ug.setSources(sourceMapping.partitions);
        }
    }
}