Examples with TriggerOption org.apache.apex.malhar.lib.window.TriggerOption used on opensource projects

Example 1 with TriggerOption

use of org.apache.apex.malhar.lib.window.TriggerOption in project apex-malhar by apache.

the class AutoComplete method populateDAG.

/**
 * Populate the dag with High-Level API.
 * @param dag
 * @param conf
 */
@Override
public void populateDAG(DAG dag, Configuration conf) {
    TweetsInput input = new TweetsInput();
    Collector collector = new Collector();
    WindowOption windowOption = new WindowOption.GlobalWindow();
    ApexStream<String> tags = StreamFactory.fromInput(input, input.output, name("tweetSampler")).flatMap(new ExtractHashtags());
    tags.window(windowOption, new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1)).addCompositeStreams(ComputeTopCompletions.top(10, true)).print(name("console")).endWith(collector, collector.input, name("collector")).populateDag(dag);
}

Example 2 with TriggerOption

use of org.apache.apex.malhar.lib.window.TriggerOption in project apex-malhar by apache.

the class TrafficRoutes method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    InfoGen infoGen = new InfoGen();
    Collector collector = new Collector();
    // Create a stream from the input operator.
    ApexStream<Tuple.TimestampedTuple<String>> stream = StreamFactory.fromInput(infoGen, infoGen.output, name("infoGen")).map(new ExtractTimestamps(), name("ExtractTimestamps"));
    stream.flatMap(new ExtractStationSpeedFn(), name("ExtractStationSpeedFn")).window(new WindowOption.SlidingTimeWindows(Duration.standardMinutes(WINDOW_DURATION), Duration.standardMinutes(WINDOW_SLIDE_EVERY)), new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(5000)).accumulatingFiredPanes()).addCompositeStreams(new TrackSpeed()).print(name("console")).endWith(collector, collector.input, name("Collector")).populateDag(dag);
}

Example 3 with TriggerOption

use of org.apache.apex.malhar.lib.window.TriggerOption in project apex-malhar by apache.

the class DeDupExample method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    Collector collector = new Collector();
    // Create a stream that reads from files in a local folder and output lines one by one to downstream.
    ApexStream<String> stream = StreamFactory.fromFolder("./src/test/resources/wordcount", name("textInput")).flatMap(new Function.FlatMapFunction<String, String>() {

        @Override
        public Iterable<String> f(String input) {
            return Arrays.asList(input.split("[\\p{Punct}\\s]+"));
        }
    }, name("ExtractWords")).map(new Function.MapFunction<String, String>() {

        @Override
        public String f(String input) {
            return input.toLowerCase();
        }
    }, name("ToLowerCase"));
    // Apply window and trigger option.
    stream.window(new WindowOption.GlobalWindow(), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(Duration.standardSeconds(1))).accumulate(new RemoveDuplicates<String>(), name("RemoveDuplicates")).print(name("console")).endWith(collector, collector.input).populateDag(dag);
}

Example 4 with TriggerOption

use of org.apache.apex.malhar.lib.window.TriggerOption in project apex-malhar by apache.

the class MaxPerKeyExamples method populateDAG.

/**
 * Populate the dag using High-Level API.
 * @param dag
 * @param conf
 */
@Override
public void populateDAG(DAG dag, Configuration conf) {
    JdbcPOJOInputOperator jdbcInput = new JdbcPOJOInputOperator();
    jdbcInput.setFieldInfos(addInputFieldInfos());
    JdbcStore store = new JdbcStore();
    jdbcInput.setStore(store);
    JdbcPOJOInsertOutputOperator jdbcOutput = new JdbcPOJOInsertOutputOperator();
    jdbcOutput.setFieldInfos(addOutputFieldInfos());
    JdbcTransactionalStore outputStore = new JdbcTransactionalStore();
    jdbcOutput.setStore(outputStore);
    // Create stream that reads from a Jdbc Input.
    ApexStream<Object> stream = StreamFactory.fromInput(jdbcInput, jdbcInput.outputPort, name("jdbcInput")).window(new WindowOption.GlobalWindow(), new TriggerOption().accumulatingFiredPanes().withEarlyFiringsAtEvery(1)).map(new Function.MapFunction<Object, InputPojo>() {

        @Override
        public InputPojo f(Object input) {
            return (InputPojo) input;
        }
    }, name("ObjectToInputPojo")).addCompositeStreams(new MaxMeanTemp()).map(new Function.MapFunction<OutputPojo, Object>() {

        @Override
        public Object f(OutputPojo input) {
            return (Object) input;
        }
    }, name("OutputPojoToObject")).endWith(jdbcOutput, jdbcOutput.input, name("jdbcOutput"));
    stream.populateDag(dag);
}

Example 5 with TriggerOption

use of org.apache.apex.malhar.lib.window.TriggerOption in project apex-malhar by apache.

the class PojoInnerJoinTestApplication method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    POJOGenerator salesGenerator = dag.addOperator("Input1", new POJOGenerator(1, 1));
    POJOGenerator productGenerator = dag.addOperator("Input2", new POJOGenerator(1, 1));
    productGenerator.setSalesEvent(false);
    WindowedMergeOperatorImpl<POJOGenerator.SalesEvent, POJOGenerator.ProductEvent, List<Set<Object>>, List<List<Object>>> op = dag.addOperator("Merge", new WindowedMergeOperatorImpl<POJOGenerator.SalesEvent, POJOGenerator.ProductEvent, List<Set<Object>>, List<List<Object>>>());
    op.setAccumulation(new PojoInnerJoin(2, POJOGenerator.OutputEvent.class, "productId", "productId"));
    op.setDataStorage(new InMemoryWindowedStorage<List<Set<Object>>>());
    WindowedStorage.WindowedPlainStorage<WindowState> windowStateMap = new InMemoryWindowedStorage<>();
    op.setWindowStateStorage(windowStateMap);
    op.setWindowOption(new WindowOption.TimeWindows(Duration.millis(10)));
    op.setTriggerOption(new TriggerOption().withEarlyFiringsAtEvery(1).accumulatingFiredPanes());
    op.setAllowedLateness(Duration.millis(500));
    ResultCollector results = dag.addOperator("results", new ResultCollector());
    dag.addStream("SalesToJoin", salesGenerator.outputsales, op.input);
    dag.addStream("ProductToJoin", productGenerator.outputproduct, op.input2);
    dag.addStream("results", op.output, results.input);
    dag.addStream("wm1", salesGenerator.watermarkDefaultOutputPort, op.controlInput);
    dag.addStream("wm2", productGenerator.watermarkDefaultOutputPort, op.controlInput2);
}