Examples with WindowState org.apache.apex.malhar.lib.window.WindowState used on opensource projects

Example 6 with WindowState

use of org.apache.apex.malhar.lib.window.WindowState in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration configuration) {
    RandomNumberPairGenerator inputOperator = new RandomNumberPairGenerator();
    WindowedOperatorImpl<MutablePair<Double, Double>, MutablePair<MutableLong, MutableLong>, Double> windowedOperator = new WindowedOperatorImpl<>();
    Accumulation<MutablePair<Double, Double>, MutablePair<MutableLong, MutableLong>, Double> piAccumulation = new PiAccumulation();
    windowedOperator.setAccumulation(piAccumulation);
    windowedOperator.setDataStorage(new InMemoryWindowedStorage<MutablePair<MutableLong, MutableLong>>());
    windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>());
    windowedOperator.setWindowOption(new WindowOption.GlobalWindow());
    windowedOperator.setTriggerOption(TriggerOption.AtWatermark().withEarlyFiringsAtEvery(Duration.millis(1000)).accumulatingFiredPanes());
    ConsoleOutputOperator outputOperator = new ConsoleOutputOperator();
    dag.addOperator("inputOperator", inputOperator);
    dag.addOperator("windowedOperator", windowedOperator);
    dag.addOperator("outputOperator", outputOperator);
    dag.addStream("input_windowed", inputOperator.output, windowedOperator.input);
    dag.addStream("windowed_output", windowedOperator.output, outputOperator.input);
}

Example 7 with WindowState

use of org.apache.apex.malhar.lib.window.WindowState in project apex-malhar by apache.

the class PojoInnerJoinTestApplication method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    POJOGenerator salesGenerator = dag.addOperator("Input1", new POJOGenerator(1, 1));
    POJOGenerator productGenerator = dag.addOperator("Input2", new POJOGenerator(1, 1));
    productGenerator.setSalesEvent(false);
    WindowedMergeOperatorImpl<POJOGenerator.SalesEvent, POJOGenerator.ProductEvent, List<Set<Object>>, List<List<Object>>> op = dag.addOperator("Merge", new WindowedMergeOperatorImpl<POJOGenerator.SalesEvent, POJOGenerator.ProductEvent, List<Set<Object>>, List<List<Object>>>());
    op.setAccumulation(new PojoInnerJoin(2, POJOGenerator.OutputEvent.class, "productId", "productId"));
    op.setDataStorage(new InMemoryWindowedStorage<List<Set<Object>>>());
    WindowedStorage.WindowedPlainStorage<WindowState> windowStateMap = new InMemoryWindowedStorage<>();
    op.setWindowStateStorage(windowStateMap);
    op.setWindowOption(new WindowOption.TimeWindows(Duration.millis(10)));
    op.setTriggerOption(new TriggerOption().withEarlyFiringsAtEvery(1).accumulatingFiredPanes());
    op.setAllowedLateness(Duration.millis(500));
    ResultCollector results = dag.addOperator("results", new ResultCollector());
    dag.addStream("SalesToJoin", salesGenerator.outputsales, op.input);
    dag.addStream("ProductToJoin", productGenerator.outputproduct, op.input2);
    dag.addStream("results", op.output, results.input);
    dag.addStream("wm1", salesGenerator.watermarkDefaultOutputPort, op.controlInput);
    dag.addStream("wm2", productGenerator.watermarkDefaultOutputPort, op.controlInput2);
}

Example 8 with WindowState

use of org.apache.apex.malhar.lib.window.WindowState in project apex-malhar by apache.

the class WindowedMergeOperatorTestApplication method assignTestWindow.

public static Window.TimeWindow assignTestWindow(long timestamp) {
    long beginTimestamp = timestamp - timestamp % windowDuration;
    Window.TimeWindow window = new Window.TimeWindow(beginTimestamp, windowDuration);
    if (!windowStateMap.containsWindow(window)) {
        windowStateMap.put(window, new WindowState());
    }
    return window;
}

Example 9 with WindowState

use of org.apache.apex.malhar.lib.window.WindowState in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration configuration) {
    WordGenerator inputOperator = new WordGenerator();
    KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> windowedOperator = new KeyedWindowedOperatorImpl<>();
    Accumulation<Long, MutableLong, Long> sum = new SumAccumulation();
    windowedOperator.setAccumulation(sum);
    windowedOperator.setDataStorage(new InMemoryWindowedKeyedStorage<String, MutableLong>());
    windowedOperator.setRetractionStorage(new InMemoryWindowedKeyedStorage<String, Long>());
    windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>());
    windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(1)));
    windowedOperator.setTriggerOption(TriggerOption.AtWatermark().withEarlyFiringsAtEvery(Duration.millis(1000)).accumulatingAndRetractingFiredPanes());
    // windowedOperator.setAllowedLateness(Duration.millis(14000));
    ConsoleOutputOperator outputOperator = new ConsoleOutputOperator();
    dag.addOperator("inputOperator", inputOperator);
    dag.addOperator("windowedOperator", windowedOperator);
    dag.addOperator("outputOperator", outputOperator);
    dag.addStream("input_windowed", inputOperator.output, windowedOperator.input);
    dag.addStream("windowed_output", windowedOperator.output, outputOperator.input);
}

Example 10 with WindowState

use of org.apache.apex.malhar.lib.window.WindowState in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, 1000);
    NycTaxiDataReader inputOperator = new NycTaxiDataReader();
    inputOperator.setDirectory("/user/" + System.getProperty("user.name") + "/nyctaxidata");
    inputOperator.getScanner().setFilePatternRegexp(".*\\.csv$");
    dag.addOperator("NycTaxiDataReader", inputOperator);
    NycTaxiCsvParser parser = dag.addOperator("NycTaxiCsvParser", new NycTaxiCsvParser());
    NycTaxiZipFareExtractor extractor = dag.addOperator("NycTaxiZipFareExtractor", new NycTaxiZipFareExtractor());
    KeyedWindowedOperatorImpl<String, Double, MutableDouble, Double> windowedOperator = new KeyedWindowedOperatorImpl<>();
    // 5-minute windows slide by 1 minute
    windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(5)).slideBy(Duration.standardMinutes(1)));
    // Because we only care about the last 5 minutes, and the watermark is set at t-1 minutes, lateness horizon is set to 4 minutes.
    windowedOperator.setAllowedLateness(Duration.standardMinutes(4));
    windowedOperator.setAccumulation(new SumDouble());
    windowedOperator.setTriggerOption(TriggerOption.AtWatermark());
    windowedOperator.setDataStorage(new InMemoryWindowedKeyedStorage<String, MutableDouble>());
    windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>());
    dag.addOperator("WindowedOperator", windowedOperator);
    NycTaxiDataServer dataServer = dag.addOperator("NycTaxiDataServer", new NycTaxiDataServer());
    ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("input_to_parser", inputOperator.output, parser.input);
    dag.addStream("parser_to_extractor", parser.output, extractor.input);
    dag.addStream("extractor_to_windowed", extractor.output, windowedOperator.input);
    dag.addStream("extractor_watermark", extractor.watermarkOutput, windowedOperator.controlInput);
    dag.addStream("windowed_to_console", windowedOperator.output, dataServer.input, console.input);
    PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
    wsQuery.enableEmbeddedMode();
    wsQuery.setTopic("nyctaxi.query");
    try {
        wsQuery.setUri(new URI("ws://" + java.net.InetAddress.getLocalHost().getHostName() + ":8890/pubsub"));
    } catch (URISyntaxException | UnknownHostException ex) {
        throw Throwables.propagate(ex);
    }
    dataServer.setEmbeddableQueryInfoProvider(wsQuery);
    PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
    wsResult.setTopic("nyctaxi.result");
    try {
        wsResult.setUri(new URI("ws://" + java.net.InetAddress.getLocalHost().getHostName() + ":8890/pubsub"));
    } catch (URISyntaxException | UnknownHostException ex) {
        throw Throwables.propagate(ex);
    }
    dag.addStream("server_to_query_output", dataServer.queryResult, wsResult.input);
}