Examples with DAG com.datatorrent.api.DAG used on opensource projects

Example 6 with DAG

use of com.datatorrent.api.DAG in project apex-core by apache.

the class LogicalPlanConfiguration method populateDAG.

/**
   * Populate the logical plan structure from properties.
   * @param dag
   */
public void populateDAG(LogicalPlan dag) {
    Configuration pconf = new Configuration(conf);
    for (final String propertyName : this.properties.stringPropertyNames()) {
        String propertyValue = this.properties.getProperty(propertyName);
        pconf.setIfUnset(propertyName, propertyValue);
    }
    AppConf appConf = this.stramConf.getChild(WILDCARD, StramElement.APPLICATION);
    if (appConf == null) {
        LOG.warn("Application configuration not found. Probably an empty app.");
        return;
    }
    Map<String, OperatorConf> operators = appConf.getChildren(StramElement.OPERATOR);
    Map<OperatorConf, GenericOperator> nodeMap = Maps.newHashMapWithExpectedSize(operators.size());
    // add all operators first
    for (Map.Entry<String, OperatorConf> nodeConfEntry : operators.entrySet()) {
        OperatorConf nodeConf = nodeConfEntry.getValue();
        if (!WILDCARD.equals(nodeConf.id)) {
            Class<? extends GenericOperator> nodeClass = StramUtils.classForName(nodeConf.getClassNameReqd(), GenericOperator.class);
            String optJson = nodeConf.getProperties().get(nodeClass.getName());
            GenericOperator operator = null;
            try {
                if (optJson != null) {
                    // if there is a special key which is the class name, it means the operator is serialized in json format
                    ObjectMapper mapper = ObjectMapperFactory.getOperatorValueDeserializer();
                    operator = mapper.readValue("{\"" + nodeClass.getName() + "\":" + optJson + "}", nodeClass);
                    addOperator(dag, nodeConfEntry.getKey(), operator);
                } else {
                    operator = addOperator(dag, nodeConfEntry.getKey(), nodeClass);
                }
                setOperatorProperties(operator, nodeConf.getProperties());
            } catch (IOException e) {
                throw new IllegalArgumentException("Error setting operator properties " + e.getMessage(), e);
            }
            nodeMap.put(nodeConf, operator);
        }
    }
    Map<String, StreamConf> streams = appConf.getChildren(StramElement.STREAM);
    // wire operators
    for (Map.Entry<String, StreamConf> streamConfEntry : streams.entrySet()) {
        StreamConf streamConf = streamConfEntry.getValue();
        DAG.StreamMeta sd = dag.addStream(streamConfEntry.getKey());
        sd.setLocality(streamConf.getLocality());
        String schemaClassName = streamConf.properties.getProperty(STREAM_SCHEMA);
        Class<?> schemaClass = null;
        if (schemaClassName != null) {
            schemaClass = StramUtils.classForName(schemaClassName, Object.class);
        }
        if (streamConf.sourceNode != null) {
            String portName = null;
            for (Map.Entry<String, StreamConf> e : streamConf.sourceNode.outputs.entrySet()) {
                if (e.getValue() == streamConf) {
                    portName = e.getKey();
                }
            }
            GenericOperator sourceDecl = nodeMap.get(streamConf.sourceNode);
            Operators.PortMappingDescriptor sourcePortMap = new Operators.PortMappingDescriptor();
            Operators.describe(sourceDecl, sourcePortMap);
            sd.setSource(sourcePortMap.outputPorts.get(portName).component);
            if (schemaClass != null) {
                dag.setOutputPortAttribute(sourcePortMap.outputPorts.get(portName).component, PortContext.TUPLE_CLASS, schemaClass);
            }
        }
        for (OperatorConf targetNode : streamConf.targetNodes) {
            String portName = null;
            for (Map.Entry<String, StreamConf> e : targetNode.inputs.entrySet()) {
                if (e.getValue() == streamConf) {
                    portName = e.getKey();
                }
            }
            GenericOperator targetDecl = nodeMap.get(targetNode);
            Operators.PortMappingDescriptor targetPortMap = new Operators.PortMappingDescriptor();
            Operators.describe(targetDecl, targetPortMap);
            sd.addSink(targetPortMap.inputPorts.get(portName).component);
            if (schemaClass != null) {
                dag.setInputPortAttribute(targetPortMap.inputPorts.get(portName).component, PortContext.TUPLE_CLASS, schemaClass);
            }
        }
    }
}

Example 7 with DAG

use of com.datatorrent.api.DAG in project beam by apache.

the class ApexRunner method run.

@Override
public ApexRunnerResult run(final Pipeline pipeline) {
    pipeline.replaceAll(getOverrides());
    final ApexPipelineTranslator translator = new ApexPipelineTranslator(options);
    final AtomicReference<DAG> apexDAG = new AtomicReference<>();
    StreamingApplication apexApp = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            apexDAG.set(dag);
            dag.setAttribute(DAGContext.APPLICATION_NAME, options.getApplicationName());
            translator.translate(pipeline, dag);
        }
    };
    Properties configProperties = new Properties();
    try {
        if (options.getConfigFile() != null) {
            URI configURL = new URI(options.getConfigFile());
            if (CLASSPATH_SCHEME.equals(configURL.getScheme())) {
                InputStream is = this.getClass().getResourceAsStream(configURL.getPath());
                if (is != null) {
                    configProperties.load(is);
                    is.close();
                }
            } else {
                if (!configURL.isAbsolute()) {
                    // resolve as local file name
                    File f = new File(options.getConfigFile());
                    configURL = f.toURI();
                }
                try (InputStream is = configURL.toURL().openStream()) {
                    configProperties.load(is);
                }
            }
        }
    } catch (IOException | URISyntaxException ex) {
        throw new RuntimeException("Error loading properties", ex);
    }
    if (options.isEmbeddedExecution()) {
        EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
        Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
        launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
        if (options.isEmbeddedExecutionDebugMode()) {
            // turns off timeout checking for operator progress
            launchAttributes.put(EmbeddedAppLauncher.HEARTBEAT_MONITORING, false);
        }
        Configuration conf = new Configuration(false);
        ApexYarnLauncher.addProperties(conf, configProperties);
        try {
            if (translateOnly) {
                launcher.prepareDAG(apexApp, conf);
                return new ApexRunnerResult(launcher.getDAG(), null);
            }
            ApexRunner.ASSERTION_ERROR.set(null);
            AppHandle apexAppResult = launcher.launchApp(apexApp, conf, launchAttributes);
            return new ApexRunnerResult(apexDAG.get(), apexAppResult);
        } catch (Exception e) {
            Throwables.throwIfUnchecked(e);
            throw new RuntimeException(e);
        }
    } else {
        try {
            ApexYarnLauncher yarnLauncher = new ApexYarnLauncher();
            AppHandle apexAppResult = yarnLauncher.launchApp(apexApp, configProperties);
            return new ApexRunnerResult(apexDAG.get(), apexAppResult);
        } catch (IOException e) {
            throw new RuntimeException("Failed to launch the application on YARN.", e);
        }
    }
}

Example 8 with DAG

use of com.datatorrent.api.DAG in project beam by apache.

the class ApexYarnLauncher method launchApp.

public AppHandle launchApp(StreamingApplication app, Properties configProperties) throws IOException {
    List<File> jarsToShip = getYarnDeployDependencies();
    StringBuilder classpath = new StringBuilder();
    for (File path : jarsToShip) {
        if (path.isDirectory()) {
            File tmpJar = File.createTempFile("beam-runners-apex-", ".jar");
            createJar(path, tmpJar);
            tmpJar.deleteOnExit();
            path = tmpJar;
        }
        if (classpath.length() != 0) {
            classpath.append(':');
        }
        classpath.append(path.getAbsolutePath());
    }
    EmbeddedAppLauncher<?> embeddedLauncher = Launcher.getLauncher(LaunchMode.EMBEDDED);
    DAG dag = embeddedLauncher.getDAG();
    app.populateDAG(dag, new Configuration(false));
    Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    launchAttributes.put(YarnAppLauncher.LIB_JARS, classpath.toString().replace(':', ','));
    LaunchParams lp = new LaunchParams(dag, launchAttributes, configProperties);
    lp.cmd = "hadoop " + ApexYarnLauncher.class.getName();
    HashMap<String, String> env = new HashMap<>();
    env.put("HADOOP_USER_CLASSPATH_FIRST", "1");
    env.put("HADOOP_CLASSPATH", classpath.toString());
    lp.env = env;
    return launchApp(lp);
}

Example 9 with DAG

use of com.datatorrent.api.DAG in project beam by apache.

the class TestApexRunner method translate.

public static DAG translate(Pipeline pipeline, ApexPipelineOptions options) {
    ApexRunner delegate = new ApexRunner(options);
    delegate.translateOnly = true;
    DAG dag = delegate.run(pipeline).getApexDAG();
    return dag;
}

Example 10 with DAG

use of com.datatorrent.api.DAG in project beam by apache.

the class TranslationContext method populateDAG.

public void populateDAG(DAG dag) {
    for (Map.Entry<String, Operator> nameAndOperator : this.operators.entrySet()) {
        dag.addOperator(nameAndOperator.getKey(), nameAndOperator.getValue());
    }
    int streamIndex = 0;
    for (Map.Entry<PCollection, Pair<OutputPortInfo, List<InputPortInfo>>> streamEntry : this.streams.entrySet()) {
        List<InputPortInfo> destInfo = streamEntry.getValue().getRight();
        InputPort[] sinks = new InputPort[destInfo.size()];
        for (int i = 0; i < sinks.length; i++) {
            sinks[i] = destInfo.get(i).port;
        }
        if (sinks.length > 0) {
            DAG.StreamMeta streamMeta = dag.addStream("stream" + streamIndex++, streamEntry.getValue().getLeft().port, sinks);
            if (pipelineOptions.isParDoFusionEnabled()) {
                optimizeStreams(streamMeta, streamEntry);
            }
            for (InputPort port : sinks) {
                PCollection pc = streamEntry.getKey();
                Coder coder = pc.getCoder();
                if (pc.getWindowingStrategy() != null) {
                    coder = FullWindowedValueCoder.of(pc.getCoder(), pc.getWindowingStrategy().getWindowFn().windowCoder());
                }
                Coder<Object> wrapperCoder = ApexStreamTuple.ApexStreamTupleCoder.of(coder);
                CoderAdapterStreamCodec streamCodec = new CoderAdapterStreamCodec(wrapperCoder);
                dag.setInputPortAttribute(port, PortContext.STREAM_CODEC, streamCodec);
            }
        }
    }
}