Examples with ConsoleOutputOperator org.apache.apex.malhar.lib.io.ConsoleOutputOperator used on opensource projects

Example 31 with ConsoleOutputOperator

use of org.apache.apex.malhar.lib.io.ConsoleOutputOperator in project apex-malhar by apache.

the class ApplicationWithQuerySupport method populateDAG.

/**
 * Populates the DAG with operators and connecting streams
 *
 * @param dag The directed acyclic graph of operators to populate
 * @param conf The configuration
 */
@Override
public void populateDAG(DAG dag, Configuration conf) {
    // create operators
    LineReader lineReader = dag.addOperator("lineReader", new LineReader());
    WordReader wordReader = dag.addOperator("wordReader", new WordReader());
    WindowWordCount windowWordCount = dag.addOperator("windowWordCount", new WindowWordCount());
    FileWordCount fileWordCount = dag.addOperator("fileWordCount", new FileWordCount());
    WordCountWriter wcWriter = dag.addOperator("wcWriter", new WordCountWriter());
    ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
    console.setStringFormat("wordCount: %s");
    // create streams
    dag.addStream("lines", lineReader.output, wordReader.input);
    dag.addStream("control", lineReader.control, fileWordCount.control);
    dag.addStream("words", wordReader.output, windowWordCount.input);
    dag.addStream("windowWordCounts", windowWordCount.output, fileWordCount.input);
    dag.addStream("fileWordCounts", fileWordCount.fileOutput, wcWriter.input);
    if (PubSubHelper.isGatewayConfigured(dag)) {
        // add query support
        URI uri = PubSubHelper.getURI(dag);
        AppDataSnapshotServerMap snapshotServerFile = dag.addOperator("snapshotServerFile", new AppDataSnapshotServerMap());
        AppDataSnapshotServerMap snapshotServerGlobal = dag.addOperator("snapshotServerGlobal", new AppDataSnapshotServerMap());
        String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
        snapshotServerFile.setSnapshotSchemaJSON(snapshotServerJSON);
        snapshotServerGlobal.setSnapshotSchemaJSON(snapshotServerJSON);
        PubSubWebSocketAppDataQuery wsQueryFile = new PubSubWebSocketAppDataQuery();
        PubSubWebSocketAppDataQuery wsQueryGlobal = new PubSubWebSocketAppDataQuery();
        wsQueryFile.setUri(uri);
        wsQueryGlobal.setUri(uri);
        snapshotServerFile.setEmbeddableQueryInfoProvider(wsQueryFile);
        snapshotServerGlobal.setEmbeddableQueryInfoProvider(wsQueryGlobal);
        PubSubWebSocketAppDataResult wsResultFile = dag.addOperator("wsResultFile", new PubSubWebSocketAppDataResult());
        PubSubWebSocketAppDataResult wsResultGlobal = dag.addOperator("wsResultGlobal", new PubSubWebSocketAppDataResult());
        wsResultFile.setUri(uri);
        wsResultGlobal.setUri(uri);
        Operator.InputPort<String> queryResultFilePort = wsResultFile.input;
        Operator.InputPort<String> queryResultGlobalPort = wsResultGlobal.input;
        dag.addStream("WordCountsFile", fileWordCount.outputPerFile, snapshotServerFile.input, console.input);
        dag.addStream("WordCountsGlobal", fileWordCount.outputGlobal, snapshotServerGlobal.input);
        dag.addStream("ResultFile", snapshotServerFile.queryResult, queryResultFilePort);
        dag.addStream("ResultGlobal", snapshotServerGlobal.queryResult, queryResultGlobalPort);
    } else {
        // throw new RuntimeException("Error: No GATEWAY_CONNECT_ADDRESS");
        dag.addStream("WordCounts", fileWordCount.outputPerFile, console.input);
    }
    LOG.info("done with populateDAG, isDebugEnabled = " + LOG.isDebugEnabled());
    LOG.info("Returning from populateDAG");
}

Example 32 with ConsoleOutputOperator

use of org.apache.apex.malhar.lib.io.ConsoleOutputOperator in project apex-malhar by apache.

the class ApplicationWithDerbySQL method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    String symbolStr = conf.get(ApplicationWithDerbySQL.class.getName() + ".tickerSymbols", "AABA,GOOG,AAPL,FB,AMZN,NFLX,IBM");
    String[] symbols = symbolStr.split(",");
    YahooFinanceCSVInputOperator input1 = dag.addOperator("input1", new YahooFinanceCSVInputOperator());
    YahooFinanceCSVInputOperator input2 = dag.addOperator("input2", new YahooFinanceCSVInputOperator());
    DerbySqlStreamOperator sqlOper = dag.addOperator("sqlOper", new DerbySqlStreamOperator());
    ConsoleOutputOperator consoleOperator = dag.addOperator("console", new ConsoleOutputOperator());
    for (String symbol : symbols) {
        input1.addSymbol(symbol);
        input2.addSymbol(symbol);
    }
    input1.addFormat("s0");
    input1.addFormat("l1");
    input2.addFormat("s0");
    input2.addFormat("e0");
    input2.addFormat("b4");
    AbstractSqlStreamOperator.InputSchema inputSchema1 = new AbstractSqlStreamOperator.InputSchema("t1");
    AbstractSqlStreamOperator.InputSchema inputSchema2 = new AbstractSqlStreamOperator.InputSchema("t2");
    // symbol
    inputSchema1.setColumnInfo("s0", "varchar(100)", true);
    // last trade
    inputSchema1.setColumnInfo("l1", "float", false);
    // symbol
    inputSchema2.setColumnInfo("s0", "varchar(100)", true);
    // EPS
    inputSchema2.setColumnInfo("e0", "float", false);
    // Book value
    inputSchema2.setColumnInfo("b4", "float", false);
    sqlOper.setInputSchema(0, inputSchema1);
    sqlOper.setInputSchema(1, inputSchema2);
    // Calculate PE Ratio and PB Ratio using SQL
    sqlOper.addExecStatementString("SELECT SESSION.t1.s0 AS symbol, SESSION.t1.l1 / SESSION.t2.e0 AS pe_ratio, SESSION.t1.l1 / SESSION.t2.b4 AS pb_ratio FROM SESSION.t1,SESSION.t2 WHERE SESSION.t1.s0 = SESSION.t2.s0");
    dag.addStream("input1_sql", input1.outputPort, sqlOper.in1);
    dag.addStream("input2_sql", input2.outputPort, sqlOper.in2);
    dag.addStream("result_console", sqlOper.result, consoleOperator.input);
}

Example 33 with ConsoleOutputOperator

use of org.apache.apex.malhar.lib.io.ConsoleOutputOperator in project apex-malhar by apache.

the class KeyedWindowedMergeOperatorTestApplication method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    KeyedWindowedMergeOperatorImpl<String, Integer, Integer, List<Set<Integer>>, List<List<Integer>>> op = dag.addOperator("Merge", new KeyedWindowedMergeOperatorImpl<String, Integer, Integer, List<Set<Integer>>, List<List<Integer>>>());
    // op.setAccumulation(new CoGroup<Integer>());
    op.setAccumulation(new InnerJoin<Integer>());
    op.setDataStorage(new InMemoryWindowedKeyedStorage<String, List<Set<Integer>>>());
    op.setRetractionStorage(new InMemoryWindowedKeyedStorage<String, List<List<Integer>>>());
    op.setWindowStateStorage(windowStateMap);
    // Can select one of the following window options, or don't select any of them.
    op.setWindowOption(new WindowOption.GlobalWindow());
    // op.setWindowOption(new WindowOption.TimeWindows(Duration.millis(4000)));
    op.setTriggerOption(new TriggerOption().withEarlyFiringsAtEvery(1).accumulatingFiredPanes());
    op.setAllowedLateness(Duration.millis(500));
    NumGen1 numGen1 = dag.addOperator("numGen1", new NumGen1());
    NumGen2 numGen2 = dag.addOperator("numGen2", new NumGen2());
    Collector collector = dag.addOperator("collector", new Collector());
    ConsoleOutputOperator con = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("num1", numGen1.output, op.input);
    dag.addStream("num2", numGen2.output, op.input2);
    dag.addStream("wm1", numGen1.watermarkDefaultOutputPort, op.controlInput);
    dag.addStream("wm2", numGen2.watermarkDefaultOutputPort, op.controlInput2);
    dag.addStream("MergedResult", op.output, collector.input);
    dag.addStream("output", collector.output, con.input);
}

Example 34 with ConsoleOutputOperator

use of org.apache.apex.malhar.lib.io.ConsoleOutputOperator in project apex-malhar by apache.

the class ExactlyOnceJdbcOutputApp method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    KafkaSinglePortStringInputOperator kafkaInput = dag.addOperator("kafkaInput", new KafkaSinglePortStringInputOperator());
    kafkaInput.setWindowDataManager(new FSWindowDataManager());
    UniqueCounterFlat count = dag.addOperator("count", new UniqueCounterFlat());
    CountStoreOperator store = dag.addOperator("store", new CountStoreOperator());
    store.setStore(new JdbcTransactionalStore());
    ConsoleOutputOperator cons = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("words", kafkaInput.outputPort, count.data);
    dag.addStream("counts", count.counts, store.input, cons.input);
}

Example 35 with ConsoleOutputOperator

use of org.apache.apex.malhar.lib.io.ConsoleOutputOperator in project apex-malhar by apache.

the class Application method wsOutput.

private InputPort<Object> wsOutput(DAG dag, String operatorName) {
    if (PubSubHelper.isGatewayConfigured(dag)) {
        String appId = "appid";
        // appId = dag.attrValue(DAG.APPLICATION_ID, null); // will be used once UI is able to pick applications from list and listen to corresponding application
        String topic = "apps.logstream." + appId + "." + operatorName;
        PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator(operatorName, new PubSubWebSocketOutputOperator<Object>());
        wsOut.setUri(PubSubHelper.getURI(dag));
        wsOut.setTopic(topic);
        return wsOut.input;
    }
    ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
    operator.setStringFormat(operatorName + ": %s");
    return operator.input;
}