Examples with StreamMeta com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta used on opensource projects

Example 21 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class StreamMapping method addInput.

public static void addInput(PTOperator target, PTOutput upstreamOut, PartitionKeys pks) {
    StreamMeta lStreamMeta = upstreamOut.logicalStream;
    PTInput input = new PTInput("<merge#" + lStreamMeta.getSource().getPortName() + ">", lStreamMeta, target, pks, upstreamOut, false);
    target.inputs.add(input);
}

Example 22 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class LogicalPlanConfigurationTest method testLoadFromPropertiesFileWithLegacyPrefix.

@Test
public void testLoadFromPropertiesFileWithLegacyPrefix() throws IOException {
    Properties props = new Properties();
    String resourcePath = "/testTopologyLegacyPrefix.properties";
    InputStream is = this.getClass().getResourceAsStream(resourcePath);
    if (is == null) {
        fail("Could not load " + resourcePath);
    }
    props.load(is);
    LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);
    LogicalPlan dag = new LogicalPlan();
    pb.populateDAG(dag);
    dag.validate();
    assertEquals("number of operators", 2, dag.getAllOperators().size());
    assertEquals("number of root operators", 1, dag.getRootOperators().size());
    StreamMeta s1 = dag.getStream("s1");
    assertNotNull(s1);
    assertTrue("s1 locality", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
    OperatorMeta o2m = dag.getOperatorMeta("o2");
    assertEquals(GenericTestOperator.class, o2m.getOperator().getClass());
    GenericTestOperator o2 = (GenericTestOperator) o2m.getOperator();
    assertEquals("myStringProperty " + o2, "myStringPropertyValue", o2.getMyStringProperty());
}

Example 23 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class LogicalPlanConfigurationTest method testLoadFromPropertiesFile.

@Test
public void testLoadFromPropertiesFile() throws IOException {
    Properties props = new Properties();
    String resourcePath = "/testTopology.properties";
    InputStream is = this.getClass().getResourceAsStream(resourcePath);
    if (is == null) {
        fail("Could not load " + resourcePath);
    }
    props.load(is);
    LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);
    LogicalPlan dag = new LogicalPlan();
    pb.populateDAG(dag);
    dag.validate();
    assertEquals("number of operator confs", 5, dag.getAllOperators().size());
    assertEquals("number of root operators", 1, dag.getRootOperators().size());
    StreamMeta s1 = dag.getStream("n1n2");
    assertNotNull(s1);
    assertTrue("n1n2 locality", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
    OperatorMeta operator3 = dag.getOperatorMeta("operator3");
    assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());
    GenericTestOperator doperator3 = (GenericTestOperator) operator3.getOperator();
    assertEquals("myStringProperty " + doperator3, "myStringPropertyValueFromTemplate", doperator3.getMyStringProperty());
    assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);
    OperatorMeta operator4 = dag.getOperatorMeta("operator4");
    GenericTestOperator doperator4 = (GenericTestOperator) operator4.getOperator();
    assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
    assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
    assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);
    StreamMeta input1 = dag.getStream("inputStream");
    assertNotNull(input1);
    Assert.assertEquals("input1 source", dag.getOperatorMeta("inputOperator"), input1.getSource().getOperatorMeta());
    Set<OperatorMeta> targetNodes = Sets.newHashSet();
    for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
        targetNodes.add(targetPort.getOperatorMeta());
    }
    Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
}

Example 24 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class LogicalPlanTest method testLocalityValidation.

@Test
public void testLocalityValidation() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    StreamMeta s1 = dag.addStream("input1.outport", input1.outport, o1.inport1).setLocality(Locality.THREAD_LOCAL);
    dag.validate();
    TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
    dag.addStream("input2.outport", input2.outport, o1.inport2);
    try {
        dag.validate();
        Assert.fail("Exception expected for " + o1);
    } catch (ValidationException ve) {
        Assert.assertThat("", ve.getMessage(), RegexMatcher.matches("Locality THREAD_LOCAL invalid for operator .* with multiple input streams .*"));
    }
    s1.setLocality(null);
    dag.validate();
}

Example 25 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class LogicalPlanSerializer method convertToProperties.

public static PropertiesConfiguration convertToProperties(LogicalPlan dag) {
    PropertiesConfiguration props = new PropertiesConfiguration();
    Collection<OperatorMeta> allOperators = dag.getAllOperators();
    for (OperatorMeta operatorMeta : allOperators) {
        String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
        Operator operator = operatorMeta.getOperator();
        props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
        BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
        @SuppressWarnings("rawtypes") Iterator entryIterator = operatorProperties.entryIterator();
        while (entryIterator.hasNext()) {
            try {
                @SuppressWarnings("unchecked") Map.Entry<String, Object> entry = (Map.Entry<String, Object>) entryIterator.next();
                if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
                    props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
                }
            } catch (Exception ex) {
                LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
            }
        }
    }
    Collection<StreamMeta> allStreams = dag.getAllStreams();
    for (StreamMeta streamMeta : allStreams) {
        String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
        OutputPortMeta source = streamMeta.getSource();
        Collection<InputPortMeta> sinks = streamMeta.getSinks();
        props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
        String sinksValue = "";
        for (InputPortMeta sink : sinks) {
            if (!sinksValue.isEmpty()) {
                sinksValue += ",";
            }
            sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
        }
        props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
        if (streamMeta.getLocality() != null) {
            props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
        }
    }
    return props;
}