Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 1 with DAG

use of com.datatorrent.api.DAG in project apex-core by apache.

the class OperatorContextTest method testInjectionOfOperatorName.

@Test
public void testInjectionOfOperatorName() throws Exception {
    StreamingApplication application = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            dag.addOperator("input", new MockInputOperator());
        }
    };
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(application, new Configuration());
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    latch.await();
    Assert.assertEquals("operator name", "input", operatorName);
    lc.shutdown();
}

Example 2 with DAG

use of com.datatorrent.api.DAG in project apex-core by apache.

the class LogicalPlanConfiguration method populateDAG.

/**
   * Populate the logical plan structure from properties.
   * @param dag
   */
public void populateDAG(LogicalPlan dag) {
    Configuration pconf = new Configuration(conf);
    for (final String propertyName : this.properties.stringPropertyNames()) {
        String propertyValue = this.properties.getProperty(propertyName);
        pconf.setIfUnset(propertyName, propertyValue);
    }
    AppConf appConf = this.stramConf.getChild(WILDCARD, StramElement.APPLICATION);
    if (appConf == null) {
        LOG.warn("Application configuration not found. Probably an empty app.");
        return;
    }
    Map<String, OperatorConf> operators = appConf.getChildren(StramElement.OPERATOR);
    Map<OperatorConf, GenericOperator> nodeMap = Maps.newHashMapWithExpectedSize(operators.size());
    // add all operators first
    for (Map.Entry<String, OperatorConf> nodeConfEntry : operators.entrySet()) {
        OperatorConf nodeConf = nodeConfEntry.getValue();
        if (!WILDCARD.equals(nodeConf.id)) {
            Class<? extends GenericOperator> nodeClass = StramUtils.classForName(nodeConf.getClassNameReqd(), GenericOperator.class);
            String optJson = nodeConf.getProperties().get(nodeClass.getName());
            GenericOperator operator = null;
            try {
                if (optJson != null) {
                    // if there is a special key which is the class name, it means the operator is serialized in json format
                    ObjectMapper mapper = ObjectMapperFactory.getOperatorValueDeserializer();
                    operator = mapper.readValue("{\"" + nodeClass.getName() + "\":" + optJson + "}", nodeClass);
                    addOperator(dag, nodeConfEntry.getKey(), operator);
                } else {
                    operator = addOperator(dag, nodeConfEntry.getKey(), nodeClass);
                }
                setOperatorProperties(operator, nodeConf.getProperties());
            } catch (IOException e) {
                throw new IllegalArgumentException("Error setting operator properties " + e.getMessage(), e);
            }
            nodeMap.put(nodeConf, operator);
        }
    }
    Map<String, StreamConf> streams = appConf.getChildren(StramElement.STREAM);
    // wire operators
    for (Map.Entry<String, StreamConf> streamConfEntry : streams.entrySet()) {
        StreamConf streamConf = streamConfEntry.getValue();
        DAG.StreamMeta sd = dag.addStream(streamConfEntry.getKey());
        sd.setLocality(streamConf.getLocality());
        String schemaClassName = streamConf.properties.getProperty(STREAM_SCHEMA);
        Class<?> schemaClass = null;
        if (schemaClassName != null) {
            schemaClass = StramUtils.classForName(schemaClassName, Object.class);
        }
        if (streamConf.sourceNode != null) {
            String portName = null;
            for (Map.Entry<String, StreamConf> e : streamConf.sourceNode.outputs.entrySet()) {
                if (e.getValue() == streamConf) {
                    portName = e.getKey();
                }
            }
            GenericOperator sourceDecl = nodeMap.get(streamConf.sourceNode);
            Operators.PortMappingDescriptor sourcePortMap = new Operators.PortMappingDescriptor();
            Operators.describe(sourceDecl, sourcePortMap);
            sd.setSource(sourcePortMap.outputPorts.get(portName).component);
            if (schemaClass != null) {
                dag.setOutputPortAttribute(sourcePortMap.outputPorts.get(portName).component, PortContext.TUPLE_CLASS, schemaClass);
            }
        }
        for (OperatorConf targetNode : streamConf.targetNodes) {
            String portName = null;
            for (Map.Entry<String, StreamConf> e : targetNode.inputs.entrySet()) {
                if (e.getValue() == streamConf) {
                    portName = e.getKey();
                }
            }
            GenericOperator targetDecl = nodeMap.get(targetNode);
            Operators.PortMappingDescriptor targetPortMap = new Operators.PortMappingDescriptor();
            Operators.describe(targetDecl, targetPortMap);
            sd.addSink(targetPortMap.inputPorts.get(portName).component);
            if (schemaClass != null) {
                dag.setInputPortAttribute(targetPortMap.inputPorts.get(portName).component, PortContext.TUPLE_CLASS, schemaClass);
            }
        }
    }
}

Example 3 with DAG

use of com.datatorrent.api.DAG in project apex-core by apache.

the class LogicalPlanConfigurationTest method testOperatorLevelAttributes.

@Test
@SuppressWarnings({ "UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes" })
public void testOperatorLevelAttributes() {
    String appName = "app1";
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            dag.addOperator("operator1", GenericTestOperator.class);
            dag.addOperator("operator2", GenericTestOperator.class);
        }
    };
    Properties props = new Properties();
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".class", app.getClass().getName());
    props.put(StreamingApplication.APEX_PREFIX + "operator.*." + OperatorContext.APPLICATION_WINDOW_COUNT.getName(), "2");
    props.put(StreamingApplication.APEX_PREFIX + "operator.*." + OperatorContext.STATS_LISTENERS.getName(), PartitionLoadWatch.class.getName());
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".operator.operator1." + OperatorContext.APPLICATION_WINDOW_COUNT.getName(), "20");
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    dagBuilder.addFromProperties(props, null);
    String appPath = app.getClass().getName().replace(".", "/") + ".class";
    LogicalPlan dag = new LogicalPlan();
    dagBuilder.prepareDAG(dag, app, appPath);
    Assert.assertEquals("", Integer.valueOf(20), dag.getOperatorMeta("operator1").getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
    Assert.assertEquals("", Integer.valueOf(2), dag.getOperatorMeta("operator2").getValue(OperatorContext.APPLICATION_WINDOW_COUNT));
    Assert.assertEquals("", PartitionLoadWatch.class, dag.getOperatorMeta("operator2").getValue(OperatorContext.STATS_LISTENERS).toArray()[0].getClass());
}

Example 4 with DAG

use of com.datatorrent.api.DAG in project apex-core by apache.

the class LogicalPlanConfigurationTest method testApplicationLevelParameter.

@Test
public void testApplicationLevelParameter() {
    String appName = "app1";
    final GenericTestOperator operator1 = new GenericTestOperator();
    final GenericTestOperator operator2 = new GenericTestOperator();
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            dag.addOperator("operator1", operator1);
            dag.addOperator("operator2", operator2);
        }
    };
    Properties props = new Properties();
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".class", app.getClass().getName());
    props.put(StreamingApplication.APEX_PREFIX + "operator.*.myStringProperty", "foo ${xyz} bar ${zzz} baz");
    props.put(StreamingApplication.APEX_PREFIX + "operator.*.booleanProperty", Boolean.TRUE.toString());
    props.put(StreamingApplication.APEX_PREFIX + "application." + appName + ".operator.operator1.myStringProperty", "apv1");
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    Configuration vars = new Configuration(false);
    vars.set("xyz", "123");
    vars.set("zzz", "456");
    dagBuilder.addFromProperties(props, vars);
    String appPath = app.getClass().getName().replace(".", "/") + ".class";
    LogicalPlan dag = new LogicalPlan();
    dagBuilder.prepareDAG(dag, app, appPath);
    Assert.assertEquals("apv1", operator1.getMyStringProperty());
    Assert.assertEquals("foo 123 bar 456 baz", operator2.getMyStringProperty());
    Assert.assertEquals(true, operator2.isBooleanProperty());
}

Example 5 with DAG

use of com.datatorrent.api.DAG in project apex-core by apache.

the class LogicalPlanConfigurationTest method testModuleUnifierLevelAttributes.

@Test
@SuppressWarnings({ "UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes" })
public void testModuleUnifierLevelAttributes() {
    class DummyOperator extends BaseOperator {

        int prop;

        public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>() {

            @Override
            public void process(Integer tuple) {
                LOG.debug(tuple.intValue() + " processed");
                output.emit(tuple);
            }
        };

        public transient DefaultOutputPort<Integer> output = new DefaultOutputPort<>();
    }
    class DummyOutputOperator extends BaseOperator {

        int prop;

        public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>() {

            @Override
            public void process(Integer tuple) {
                LOG.debug(tuple.intValue() + " processed");
            }
        };
    }
    class TestUnifierAttributeModule implements Module {

        public transient ProxyInputPort<Integer> moduleInput = new ProxyInputPort<>();

        public transient ProxyOutputPort<Integer> moduleOutput = new Module.ProxyOutputPort<>();

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            DummyOperator dummyOperator = dag.addOperator("DummyOperator", new DummyOperator());
            dag.setOperatorAttribute(dummyOperator, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<DummyOperator>(3));
            dag.setUnifierAttribute(dummyOperator.output, OperatorContext.TIMEOUT_WINDOW_COUNT, 2);
            moduleInput.set(dummyOperator.input);
            moduleOutput.set(dummyOperator.output);
        }
    }
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            Module m1 = dag.addModule("TestModule", new TestUnifierAttributeModule());
            DummyOutputOperator dummyOutputOperator = dag.addOperator("DummyOutputOperator", new DummyOutputOperator());
            dag.addStream("Module To Operator", ((TestUnifierAttributeModule) m1).moduleOutput, dummyOutputOperator.input);
        }
    };
    String appName = "UnifierApp";
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new MockStorageAgent());
    dagBuilder.prepareDAG(dag, app, appName);
    LogicalPlan.OperatorMeta ometa = dag.getOperatorMeta("TestModule$DummyOperator");
    LogicalPlan.OperatorMeta om = null;
    for (Map.Entry<LogicalPlan.OutputPortMeta, LogicalPlan.StreamMeta> entry : ometa.getOutputStreams().entrySet()) {
        if (entry.getKey().getPortName().equals("output")) {
            om = entry.getKey().getUnifierMeta();
        }
    }
    /*
     * Verify the attribute value after preparing DAG.
     */
    Assert.assertNotNull(om);
    Assert.assertEquals("", Integer.valueOf(2), om.getValue(Context.OperatorContext.TIMEOUT_WINDOW_COUNT));
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    List<PTContainer> containers = plan.getContainers();
    LogicalPlan.OperatorMeta operatorMeta = null;
    for (PTContainer container : containers) {
        List<PTOperator> operators = container.getOperators();
        for (PTOperator operator : operators) {
            if (operator.isUnifier()) {
                operatorMeta = operator.getOperatorMeta();
            }
        }
    }
    /*
     * Verify attribute after physical plan creation with partitioned operators.
     */
    Assert.assertEquals("", Integer.valueOf(2), operatorMeta.getValue(OperatorContext.TIMEOUT_WINDOW_COUNT));
}