Examples with TriggerOption org.apache.apex.malhar.lib.window.TriggerOption used on opensource projects

Example 6 with TriggerOption

use of org.apache.apex.malhar.lib.window.TriggerOption in project apex-malhar by apache.

the class WindowedMergeOperatorTestApplication method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    WindowedMergeOperatorImpl<Integer, Integer, List<Set<Integer>>, List<List<Integer>>> op = dag.addOperator("Merge", new WindowedMergeOperatorImpl<Integer, Integer, List<Set<Integer>>, List<List<Integer>>>());
    op.setAccumulation(new CoGroup<Integer>());
    op.setDataStorage(new InMemoryWindowedStorage<List<Set<Integer>>>());
    op.setRetractionStorage(new InMemoryWindowedStorage<List<List<Integer>>>());
    op.setWindowStateStorage(windowStateMap);
    // Can select one of the following window options, or don't select any of them.
    // op.setWindowOption(new WindowOption.GlobalWindow());
    op.setWindowOption(new WindowOption.TimeWindows(Duration.millis(2000)));
    op.setTriggerOption(new TriggerOption().withEarlyFiringsAtEvery(1).accumulatingFiredPanes());
    op.setAllowedLateness(Duration.millis(500));
    NumGen1 numGen1 = dag.addOperator("numGen1", new NumGen1());
    NumGen2 numGen2 = dag.addOperator("numGen2", new NumGen2());
    Collector collector = dag.addOperator("collector", new Collector());
    ConsoleOutputOperator con = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("num1", numGen1.output, op.input);
    dag.addStream("num2", numGen2.output, op.input2);
    dag.addStream("wm1", numGen1.watermarkDefaultOutputPort, op.controlInput);
    dag.addStream("wm2", numGen2.watermarkDefaultOutputPort, op.controlInput2);
    dag.addStream("MergedResult", op.output, collector.input);
    dag.addStream("output", collector.output, con.input);
}

Example 7 with TriggerOption

use of org.apache.apex.malhar.lib.window.TriggerOption in project apex-malhar by apache.

the class ApplicationWithStreamAPI method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration configuration) {
    String localFolder = "./src/test/resources/data";
    ApexStream<String> stream = StreamFactory.fromFolder(localFolder).flatMap(new Function.FlatMapFunction<String, String>() {

        @Override
        public Iterable<String> f(String input) {
            return Arrays.asList(input.split("[\\p{Punct}\\s]+"));
        }
    });
    stream.print(name("WordOutput"));
    stream.window(new WindowOption.GlobalWindow(), new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(1000)).accumulatingFiredPanes()).countByKey(new Function.ToKeyValue<String, String, Long>() {

        @Override
        public Tuple<KeyValPair<String, Long>> f(String input) {
            return new Tuple.PlainTuple(new KeyValPair<>(input, 1L));
        }
    }).print(name("WCOutput"));
    stream.populateDag(dag);
}

Example 8 with TriggerOption

use of org.apache.apex.malhar.lib.window.TriggerOption in project apex-malhar by apache.

the class WordCountWithStreamAPI method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration configuration) {
    WCInput wcInput = new WCInput();
    ApexStream<String> stream = StreamFactory.fromInput(wcInput, wcInput.output).flatMap(new Function.FlatMapFunction<String, String>() {

        @Override
        public Iterable<String> f(String input) {
            return Arrays.asList(input.split("[\\p{Punct}\\s]+"));
        }
    });
    stream.print();
    stream.window(new WindowOption.GlobalWindow(), new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(1000)).accumulatingFiredPanes()).countByKey(new Function.ToKeyValue<String, String, Long>() {

        @Override
        public Tuple<KeyValPair<String, Long>> f(String input) {
            return new Tuple.PlainTuple(new KeyValPair<>(input, 1L));
        }
    }).print();
    stream.populateDag(dag);
}

Example 9 with TriggerOption

use of org.apache.apex.malhar.lib.window.TriggerOption in project apex-malhar by apache.

the class KeyedWindowedMergeOperatorTestApplication method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    KeyedWindowedMergeOperatorImpl<String, Integer, Integer, List<Set<Integer>>, List<List<Integer>>> op = dag.addOperator("Merge", new KeyedWindowedMergeOperatorImpl<String, Integer, Integer, List<Set<Integer>>, List<List<Integer>>>());
    // op.setAccumulation(new CoGroup<Integer>());
    op.setAccumulation(new InnerJoin<Integer>());
    op.setDataStorage(new InMemoryWindowedKeyedStorage<String, List<Set<Integer>>>());
    op.setRetractionStorage(new InMemoryWindowedKeyedStorage<String, List<List<Integer>>>());
    op.setWindowStateStorage(windowStateMap);
    // Can select one of the following window options, or don't select any of them.
    op.setWindowOption(new WindowOption.GlobalWindow());
    // op.setWindowOption(new WindowOption.TimeWindows(Duration.millis(4000)));
    op.setTriggerOption(new TriggerOption().withEarlyFiringsAtEvery(1).accumulatingFiredPanes());
    op.setAllowedLateness(Duration.millis(500));
    NumGen1 numGen1 = dag.addOperator("numGen1", new NumGen1());
    NumGen2 numGen2 = dag.addOperator("numGen2", new NumGen2());
    Collector collector = dag.addOperator("collector", new Collector());
    ConsoleOutputOperator con = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("num1", numGen1.output, op.input);
    dag.addStream("num2", numGen2.output, op.input2);
    dag.addStream("wm1", numGen1.watermarkDefaultOutputPort, op.controlInput);
    dag.addStream("wm2", numGen2.watermarkDefaultOutputPort, op.controlInput2);
    dag.addStream("MergedResult", op.output, collector.input);
    dag.addStream("output", collector.output, con.input);
}

Example 10 with TriggerOption

use of org.apache.apex.malhar.lib.window.TriggerOption in project apex-malhar by apache.

the class WindowedWordCount method populateDAG.

/**
 * Populate dag with High-Level API.
 * @param dag
 * @param conf
 */
@Override
public void populateDAG(DAG dag, Configuration conf) {
    TextInput input = new TextInput();
    Collector collector = new Collector();
    // Create stream from the TextInput operator.
    ApexStream<Tuple.TimestampedTuple<String>> stream = StreamFactory.fromInput(input, input.output, name("input")).flatMap(new Function.FlatMapFunction<String, String>() {

        @Override
        public Iterable<String> f(String input) {
            return Arrays.asList(input.split("[\\p{Punct}\\s]+"));
        }
    }, name("ExtractWords")).map(new AddTimestampFn(), name("AddTimestampFn"));
    // apply window and trigger option.
    // TODO: change trigger option to atWaterMark when available.
    WindowedStream<Tuple.TimestampedTuple<String>> windowedWords = stream.window(new WindowOption.TimeWindows(Duration.standardMinutes(WINDOW_SIZE)), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1));
    WindowedStream<PojoEvent> wordCounts = // Perform a countByKey transformation to count the appearance of each word in every time window.
    windowedWords.countByKey(new Function.ToKeyValue<Tuple.TimestampedTuple<String>, String, Long>() {

        @Override
        public Tuple<KeyValPair<String, Long>> f(Tuple.TimestampedTuple<String> input) {
            return new Tuple.TimestampedTuple<KeyValPair<String, Long>>(input.getTimestamp(), new KeyValPair<String, Long>(input.getValue(), 1L));
        }
    }, name("count words")).map(new FormatAsTableRowFn(), name("FormatAsTableRowFn")).print(name("console"));
    wordCounts.endWith(collector, collector.input, name("Collector")).populateDag(dag);
}