Examples with OutputPortMeta com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta used on opensource projects

Example 1 with OutputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta in project apex-core by apache.

the class StreamingContainerManager method getPortAttributes.

public Attribute.AttributeMap getPortAttributes(String operatorId, String portName) {
    OperatorMeta logicalOperator = plan.getLogicalPlan().getOperatorMeta(operatorId);
    if (logicalOperator == null) {
        throw new IllegalArgumentException("Invalid operatorId " + operatorId);
    }
    Operators.PortMappingDescriptor portMap = new Operators.PortMappingDescriptor();
    Operators.describe(logicalOperator.getOperator(), portMap);
    PortContextPair<InputPort<?>> inputPort = portMap.inputPorts.get(portName);
    if (inputPort != null) {
        InputPortMeta portMeta = logicalOperator.getMeta(inputPort.component);
        try {
            return portMeta.getAttributes().clone();
        } catch (CloneNotSupportedException ex) {
            throw new RuntimeException("Cannot clone port attributes", ex);
        }
    } else {
        PortContextPair<OutputPort<?>> outputPort = portMap.outputPorts.get(portName);
        if (outputPort != null) {
            OutputPortMeta portMeta = logicalOperator.getMeta(outputPort.component);
            try {
                return portMeta.getAttributes().clone();
            } catch (CloneNotSupportedException ex) {
                throw new RuntimeException("Cannot clone port attributes", ex);
            }
        }
        throw new IllegalArgumentException("Invalid port name " + portName);
    }
}

Example 2 with OutputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta in project apex-core by apache.

the class LogicalPlanConfiguration method setOperatorConfiguration.

private void setOperatorConfiguration(final LogicalPlan dag, List<AppConf> appConfs, String appName) {
    for (final OperatorMeta ow : dag.getAllOperators()) {
        List<OperatorConf> opConfs = getMatchingChildConf(appConfs, ow.getName(), StramElement.OPERATOR);
        // Set the operator attributes
        setAttributes(opConfs, ow.getAttributes());
        // Set the operator opProps
        Map<String, String> opProps = getProperties(getPropertyArgs(ow), opConfs, appName);
        setOperatorProperties(ow.getOperator(), opProps);
        // Set the port attributes
        for (Entry<LogicalPlan.InputPortMeta, LogicalPlan.StreamMeta> entry : ow.getInputStreams().entrySet()) {
            final InputPortMeta im = entry.getKey();
            List<PortConf> inPortConfs = getMatchingChildConf(opConfs, im.getPortName(), StramElement.INPUT_PORT);
            // Add the generic port attributes as well
            List<PortConf> portConfs = getMatchingChildConf(opConfs, im.getPortName(), StramElement.PORT);
            inPortConfs.addAll(portConfs);
            setAttributes(inPortConfs, im.getAttributes());
        }
        for (Entry<LogicalPlan.OutputPortMeta, LogicalPlan.StreamMeta> entry : ow.getOutputStreams().entrySet()) {
            final OutputPortMeta om = entry.getKey();
            List<PortConf> outPortConfs = getMatchingChildConf(opConfs, om.getPortName(), StramElement.OUTPUT_PORT);
            // Add the generic port attributes as well
            List<PortConf> portConfs = getMatchingChildConf(opConfs, om.getPortName(), StramElement.PORT);
            outPortConfs.addAll(portConfs);
            setAttributes(outPortConfs, om.getAttributes());
            List<OperatorConf> unifConfs = getMatchingChildConf(outPortConfs, null, StramElement.UNIFIER);
            if (unifConfs.size() != 0) {
                setAttributes(unifConfs, om.getUnifierMeta().getAttributes());
            }
        }
        ow.populateAggregatorMeta();
    }
}

Example 3 with OutputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta in project apex-core by apache.

the class PhysicalPlan method updateStreamMappings.

private void updateStreamMappings(PMapping m) {
    for (Map.Entry<OutputPortMeta, StreamMeta> opm : m.logicalOperator.getOutputStreams().entrySet()) {
        StreamMapping ug = m.outputStreams.get(opm.getKey());
        if (ug == null) {
            ug = new StreamMapping(opm.getValue(), this);
            m.outputStreams.put(opm.getKey(), ug);
        }
        LOG.debug("update stream mapping for {} {}", opm.getKey().getOperatorMeta(), opm.getKey().getPortName());
        ug.setSources(m.partitions);
    }
    for (Map.Entry<InputPortMeta, StreamMeta> ipm : m.logicalOperator.getInputStreams().entrySet()) {
        PMapping sourceMapping = this.logicalToPTOperator.get(ipm.getValue().getSource().getOperatorMeta());
        if (ipm.getValue().getSource().getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
            // skip if the source is a DelayOperator
            continue;
        }
        if (ipm.getKey().getValue(PortContext.PARTITION_PARALLEL)) {
            if (sourceMapping.partitions.size() < m.partitions.size()) {
                throw new AssertionError("Number of partitions don't match in parallel mapping " + sourceMapping.logicalOperator.getName() + " -> " + m.logicalOperator.getName() + ", " + sourceMapping.partitions.size() + " -> " + m.partitions.size());
            }
            int slidingWindowCount = 0;
            OperatorMeta sourceOM = sourceMapping.logicalOperator;
            if (sourceOM.getAttributes().contains(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT)) {
                if (sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT) < sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT)) {
                    slidingWindowCount = sourceOM.getValue(OperatorContext.SLIDE_BY_WINDOW_COUNT);
                } else {
                    LOG.warn("Sliding Window Count {} should be less than APPLICATION WINDOW COUNT {}", sourceOM.getValue(Context.OperatorContext.SLIDE_BY_WINDOW_COUNT), sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT));
                }
            }
            for (int i = 0; i < m.partitions.size(); i++) {
                PTOperator oper = m.partitions.get(i);
                PTOperator sourceOper = sourceMapping.partitions.get(i);
                for (PTOutput sourceOut : sourceOper.outputs) {
                    nextSource: if (sourceOut.logicalStream == ipm.getValue()) {
                        //avoid duplicate entries in case of parallel partitions
                        for (PTInput sinkIn : sourceOut.sinks) {
                            // input-port-meta currently being looked at since we allow an output port to connect to multiple inputs of the same operator.
                            if (sinkIn.target == oper && sinkIn.portName.equals(ipm.getKey().getPortName())) {
                                break nextSource;
                            }
                        }
                        PTInput input;
                        if (slidingWindowCount > 0) {
                            PTOperator slidingUnifier = StreamMapping.createSlidingUnifier(sourceOut.logicalStream, this, sourceOM.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT), slidingWindowCount);
                            StreamMapping.addInput(slidingUnifier, sourceOut, null);
                            input = new PTInput(ipm.getKey().getPortName(), ipm.getValue(), oper, null, slidingUnifier.outputs.get(0), ipm.getKey().getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR));
                            sourceMapping.outputStreams.get(ipm.getValue().getSource()).slidingUnifiers.add(slidingUnifier);
                        } else {
                            input = new PTInput(ipm.getKey().getPortName(), ipm.getValue(), oper, null, sourceOut, ipm.getKey().getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR));
                        }
                        oper.inputs.add(input);
                    }
                }
            }
        } else {
            StreamMapping ug = sourceMapping.outputStreams.get(ipm.getValue().getSource());
            if (ug == null) {
                ug = new StreamMapping(ipm.getValue(), this);
                m.outputStreams.put(ipm.getValue().getSource(), ug);
            }
            LOG.debug("update upstream stream mapping for {} {}", sourceMapping.logicalOperator, ipm.getValue().getSource().getPortName());
            ug.setSources(sourceMapping.partitions);
        }
    }
}

Example 4 with OutputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta in project apex-core by apache.

the class LogicalPlanConfigurationTest method testUnifierLevelAttributes.

@Test
@SuppressWarnings({ "UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes" })
public void testUnifierLevelAttributes() {
    String appName = "app1";
    final GenericTestOperator operator1 = new GenericTestOperator();
    final GenericTestOperator operator2 = new GenericTestOperator();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            dag.addOperator("operator1", operator1);
            dag.addOperator("operator2", operator2);
            dag.addStream("s1", operator1.outport1, operator2.inport1);
        }
    };
    Properties props = new Properties();
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".class", app.getClass().getName());
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".operator.operator1.outputport.outport1.unifier." + OperatorContext.APPLICATION_WINDOW_COUNT.getName(), "2");
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".operator.operator1.outputport.outport1.unifier." + OperatorContext.MEMORY_MB.getName(), "512");
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    dagBuilder.addFromProperties(props, null);
    String appPath = app.getClass().getName().replace(".", "/") + ".class";
    LogicalPlan dag = new LogicalPlan();
    dagBuilder.prepareDAG(dag, app, appPath);
    OperatorMeta om = null;
    for (Map.Entry<OutputPortMeta, StreamMeta> entry : dag.getOperatorMeta("operator1").getOutputStreams().entrySet()) {
        if (entry.getKey().getPortName().equals("outport1")) {
            om = entry.getKey().getUnifierMeta();
        }
    }
    Assert.assertNotNull(om);
    Assert.assertEquals("", Integer.valueOf(2), om.getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
    Assert.assertEquals("", Integer.valueOf(512), om.getValue(OperatorContext.MEMORY_MB));
}

Example 5 with OutputPortMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta in project apex-core by apache.

the class LogicalPlanConfigurationTest method testLoadFromJson.

@Test
public void testLoadFromJson() throws Exception {
    String resourcePath = "/testTopology.json";
    InputStream is = this.getClass().getResourceAsStream(resourcePath);
    if (is == null) {
        fail("Could not load " + resourcePath);
    }
    StringWriter writer = new StringWriter();
    IOUtils.copy(is, writer);
    JSONObject json = new JSONObject(writer.toString());
    Configuration conf = new Configuration(false);
    conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");
    LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
    LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
    dag.validate();
    assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
    Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
    stringCodecsMap.put(Integer.class, Integer2String.class);
    assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
    assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());
    assertEquals("number of operator confs", 5, dag.getAllOperators().size());
    assertEquals("number of root operators", 1, dag.getRootOperators().size());
    StreamMeta s1 = dag.getStream("n1n2");
    assertNotNull(s1);
    assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
    OperatorMeta input = dag.getOperatorMeta("inputOperator");
    TestStatsListener tsl = new TestStatsListener();
    tsl.setIntProp(222);
    List<StatsListener> sll = Lists.<StatsListener>newArrayList(tsl);
    assertEquals("inputOperator STATS_LISTENERS attribute ", sll, input.getAttributes().get(OperatorContext.STATS_LISTENERS));
    for (OutputPortMeta opm : input.getOutputStreams().keySet()) {
        assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
    }
    OperatorMeta operator3 = dag.getOperatorMeta("operator3");
    assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());
    GenericTestOperator doperator3 = (GenericTestOperator) operator3.getOperator();
    assertEquals("myStringProperty " + doperator3, "o3StringFromConf", doperator3.getMyStringProperty());
    assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);
    OperatorMeta operator4 = dag.getOperatorMeta("operator4");
    GenericTestOperator doperator4 = (GenericTestOperator) operator4.getOperator();
    assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
    assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
    assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);
    StreamMeta input1 = dag.getStream("inputStream");
    assertNotNull(input1);
    OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
    Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
    Set<OperatorMeta> targetNodes = Sets.newHashSet();
    for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
        targetNodes.add(targetPort.getOperatorMeta());
    }
    Assert.assertEquals("operator attribute " + inputOperator, 64, (int) inputOperator.getValue(OperatorContext.MEMORY_MB));
    Assert.assertEquals("port attribute " + inputOperator, 8, (int) input1.getSource().getValue(PortContext.UNIFIER_LIMIT));
    Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
}