Examples with ConsoleOutputOperator org.apache.apex.malhar.lib.io.ConsoleOutputOperator used on opensource projects

Example 21 with ConsoleOutputOperator

use of org.apache.apex.malhar.lib.io.ConsoleOutputOperator in project apex-malhar by apache.

the class StatefulApplication method populateDAG.

@SuppressWarnings("unchecked")
@Override
public void populateDAG(DAG dag, Configuration conf) {
    RandomKeyValGenerator randGen = dag.addOperator("RandomGenerator", new RandomKeyValGenerator());
    UniqueValueCount<Integer> valCount = dag.addOperator("UniqueCounter", new UniqueValueCount<Integer>());
    ConsoleOutputOperator consOut = dag.addOperator("Console", new ConsoleOutputOperator());
    IntegerUniqueValueCountAppender uniqueUnifier = dag.addOperator("StatefulUniqueCounter", new IntegerUniqueValueCountAppender());
    dag.getOperatorMeta("StatefulUniqueCounter").getMeta(uniqueUnifier.input).getAttributes().put(Context.PortContext.STREAM_CODEC, new KeyBasedStreamCodec());
    @SuppressWarnings("rawtypes") DefaultOutputPort valOut = valCount.output;
    @SuppressWarnings("rawtypes") DefaultOutputPort uniqueOut = uniqueUnifier.output;
    dag.addStream("Events", randGen.outport, valCount.input);
    dag.addStream("Unified", valOut, uniqueUnifier.input);
    dag.addStream("Result", uniqueOut, consOut.input);
}

Example 22 with ConsoleOutputOperator

use of org.apache.apex.malhar.lib.io.ConsoleOutputOperator in project apex-malhar by apache.

the class ApplicationAppData method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
    PiCalculateOperator calc = dag.addOperator("picalc", new PiCalculateOperator());
    dag.addStream("rand_calc", rand.integer_data, calc.input).setLocality(locality);
    AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());
    String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
    snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
    PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
    wsQuery.enableEmbeddedMode();
    snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);
    PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
    Operator.InputPort<String> queryResultPort = wsResult.input;
    NamedValueList<Object> adaptor = dag.addOperator("adaptor", new NamedValueList<Object>());
    ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("PiValues", calc.output, adaptor.inPort, console.input).setLocality(locality);
    ;
    dag.addStream("NamedPiValues", adaptor.outPort, snapshotServer.input);
    dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
}

Example 23 with ConsoleOutputOperator

use of org.apache.apex.malhar.lib.io.ConsoleOutputOperator in project apex-malhar by apache.

the class Calculator method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    /* keep generating random values between 0 and 30000 */
    RandomEventGenerator xyGenerator = dag.addOperator("GenerateX", RandomEventGenerator.class);
    /* calculate square of each of the values it receives */
    SquareCalculus squareOperator = dag.addOperator("SquareX", SquareCalculus.class);
    /* pair the consecutive values */
    AbstractAggregator<Integer> pairOperator = dag.addOperator("PairXY", new ArrayListAggregator<Integer>());
    Sigma<Integer> sumOperator = dag.addOperator("SumXY", new Sigma<Integer>());
    LogicalCompareToConstant<Integer> comparator = dag.addOperator("AnalyzeLocation", new LogicalCompareToConstant<Integer>());
    comparator.setConstant(30000 * 30000);
    Counter inCircle = dag.addOperator("CountInCircle", Counter.class);
    Counter inSquare = dag.addOperator("CountInSquare", Counter.class);
    Division division = dag.addOperator("Ratio", Division.class);
    MultiplyByConstant multiplication = dag.addOperator("InstantPI", MultiplyByConstant.class);
    multiplication.setMultiplier(4);
    RunningAverage average = dag.addOperator("AveragePI", new RunningAverage());
    ConsoleOutputOperator oper = dag.addOperator("Console", new ConsoleOutputOperator());
    dag.addStream("x", xyGenerator.integer_data, squareOperator.input);
    dag.addStream("sqr", squareOperator.integerResult, pairOperator.input);
    dag.addStream("x2andy2", pairOperator.output, sumOperator.input);
    dag.addStream("x2plusy2", sumOperator.integerResult, comparator.input, inSquare.input);
    dag.addStream("inCirclePoints", comparator.greaterThan, inCircle.input);
    dag.addStream("numerator", inCircle.output, division.numerator);
    dag.addStream("denominator", inSquare.output, division.denominator);
    dag.addStream("ratio", division.doubleQuotient, multiplication.input);
    dag.addStream("instantPi", multiplication.doubleProduct, average.input);
    dag.addStream("averagePi", average.doubleAverage, oper.input);
}

Example 24 with ConsoleOutputOperator

use of org.apache.apex.malhar.lib.io.ConsoleOutputOperator in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, 1000);
    NycTaxiDataReader inputOperator = new NycTaxiDataReader();
    inputOperator.setDirectory("/user/" + System.getProperty("user.name") + "/nyctaxidata");
    inputOperator.getScanner().setFilePatternRegexp(".*\\.csv$");
    dag.addOperator("NycTaxiDataReader", inputOperator);
    NycTaxiCsvParser parser = dag.addOperator("NycTaxiCsvParser", new NycTaxiCsvParser());
    NycTaxiZipFareExtractor extractor = dag.addOperator("NycTaxiZipFareExtractor", new NycTaxiZipFareExtractor());
    KeyedWindowedOperatorImpl<String, Double, MutableDouble, Double> windowedOperator = new KeyedWindowedOperatorImpl<>();
    // 5-minute windows slide by 1 minute
    windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(5)).slideBy(Duration.standardMinutes(1)));
    // Because we only care about the last 5 minutes, and the watermark is set at t-1 minutes, lateness horizon is set to 4 minutes.
    windowedOperator.setAllowedLateness(Duration.standardMinutes(4));
    windowedOperator.setAccumulation(new SumDouble());
    windowedOperator.setTriggerOption(TriggerOption.AtWatermark());
    windowedOperator.setDataStorage(new InMemoryWindowedKeyedStorage<String, MutableDouble>());
    windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>());
    dag.addOperator("WindowedOperator", windowedOperator);
    NycTaxiDataServer dataServer = dag.addOperator("NycTaxiDataServer", new NycTaxiDataServer());
    ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("input_to_parser", inputOperator.output, parser.input);
    dag.addStream("parser_to_extractor", parser.output, extractor.input);
    dag.addStream("extractor_to_windowed", extractor.output, windowedOperator.input);
    dag.addStream("extractor_watermark", extractor.watermarkOutput, windowedOperator.controlInput);
    dag.addStream("windowed_to_console", windowedOperator.output, dataServer.input, console.input);
    PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
    wsQuery.enableEmbeddedMode();
    wsQuery.setTopic("nyctaxi.query");
    try {
        wsQuery.setUri(new URI("ws://" + java.net.InetAddress.getLocalHost().getHostName() + ":8890/pubsub"));
    } catch (URISyntaxException | UnknownHostException ex) {
        throw Throwables.propagate(ex);
    }
    dataServer.setEmbeddableQueryInfoProvider(wsQuery);
    PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
    wsResult.setTopic("nyctaxi.result");
    try {
        wsResult.setUri(new URI("ws://" + java.net.InetAddress.getLocalHost().getHostName() + ":8890/pubsub"));
    } catch (URISyntaxException | UnknownHostException ex) {
        throw Throwables.propagate(ex);
    }
    dag.addStream("server_to_query_output", dataServer.queryResult, wsResult.input);
}

Example 25 with ConsoleOutputOperator

use of org.apache.apex.malhar.lib.io.ConsoleOutputOperator in project apex-malhar by apache.

the class KinesisHashtagsApplication method consoleOutput.

private InputPort<Object> consoleOutput(DAG dag, String operatorName) {
    if (PubSubHelper.isGatewayConfigured(dag)) {
        String topic = "examples.twitter." + operatorName;
        // LOG.info("WebSocket with gateway at: {}", gatewayAddress);
        PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator(operatorName, new PubSubWebSocketOutputOperator<Object>());
        wsOut.setUri(PubSubHelper.getURI(dag));
        wsOut.setTopic(topic);
        return wsOut.input;
    }
    ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
    operator.setStringFormat(operatorName + ": %s");
    return operator.input;
}