Examples with PubSubWebSocketAppDataQuery org.apache.apex.malhar.lib.io.PubSubWebSocketAppDataQuery used on opensource projects

Example 1 with PubSubWebSocketAppDataQuery

use of org.apache.apex.malhar.lib.io.PubSubWebSocketAppDataQuery in project apex-malhar by apache.

the class ApplicationAppData method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
    PiCalculateOperator calc = dag.addOperator("picalc", new PiCalculateOperator());
    dag.addStream("rand_calc", rand.integer_data, calc.input).setLocality(locality);
    AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());
    String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
    snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
    PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
    wsQuery.enableEmbeddedMode();
    snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);
    PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
    Operator.InputPort<String> queryResultPort = wsResult.input;
    NamedValueList<Object> adaptor = dag.addOperator("adaptor", new NamedValueList<Object>());
    ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("PiValues", calc.output, adaptor.inPort, console.input).setLocality(locality);
    ;
    dag.addStream("NamedPiValues", adaptor.outPort, snapshotServer.input);
    dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
}

Example 2 with PubSubWebSocketAppDataQuery

use of org.apache.apex.malhar.lib.io.PubSubWebSocketAppDataQuery in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, 1000);
    NycTaxiDataReader inputOperator = new NycTaxiDataReader();
    inputOperator.setDirectory("/user/" + System.getProperty("user.name") + "/nyctaxidata");
    inputOperator.getScanner().setFilePatternRegexp(".*\\.csv$");
    dag.addOperator("NycTaxiDataReader", inputOperator);
    NycTaxiCsvParser parser = dag.addOperator("NycTaxiCsvParser", new NycTaxiCsvParser());
    NycTaxiZipFareExtractor extractor = dag.addOperator("NycTaxiZipFareExtractor", new NycTaxiZipFareExtractor());
    KeyedWindowedOperatorImpl<String, Double, MutableDouble, Double> windowedOperator = new KeyedWindowedOperatorImpl<>();
    // 5-minute windows slide by 1 minute
    windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(5)).slideBy(Duration.standardMinutes(1)));
    // Because we only care about the last 5 minutes, and the watermark is set at t-1 minutes, lateness horizon is set to 4 minutes.
    windowedOperator.setAllowedLateness(Duration.standardMinutes(4));
    windowedOperator.setAccumulation(new SumDouble());
    windowedOperator.setTriggerOption(TriggerOption.AtWatermark());
    windowedOperator.setDataStorage(new InMemoryWindowedKeyedStorage<String, MutableDouble>());
    windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>());
    dag.addOperator("WindowedOperator", windowedOperator);
    NycTaxiDataServer dataServer = dag.addOperator("NycTaxiDataServer", new NycTaxiDataServer());
    ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
    dag.addStream("input_to_parser", inputOperator.output, parser.input);
    dag.addStream("parser_to_extractor", parser.output, extractor.input);
    dag.addStream("extractor_to_windowed", extractor.output, windowedOperator.input);
    dag.addStream("extractor_watermark", extractor.watermarkOutput, windowedOperator.controlInput);
    dag.addStream("windowed_to_console", windowedOperator.output, dataServer.input, console.input);
    PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
    wsQuery.enableEmbeddedMode();
    wsQuery.setTopic("nyctaxi.query");
    try {
        wsQuery.setUri(new URI("ws://" + java.net.InetAddress.getLocalHost().getHostName() + ":8890/pubsub"));
    } catch (URISyntaxException | UnknownHostException ex) {
        throw Throwables.propagate(ex);
    }
    dataServer.setEmbeddableQueryInfoProvider(wsQuery);
    PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
    wsResult.setTopic("nyctaxi.result");
    try {
        wsResult.setUri(new URI("ws://" + java.net.InetAddress.getLocalHost().getHostName() + ":8890/pubsub"));
    } catch (URISyntaxException | UnknownHostException ex) {
        throw Throwables.propagate(ex);
    }
    dag.addStream("server_to_query_output", dataServer.queryResult, wsResult.input);
}

Example 3 with PubSubWebSocketAppDataQuery

use of org.apache.apex.malhar.lib.io.PubSubWebSocketAppDataQuery in project apex-malhar by apache.

the class TwitterTopCounterApplication method consoleOutput.

public static void consoleOutput(DAG dag, String operatorName, OutputPort<List<Map<String, Object>>> topCount, String schemaFile, String alias) {
    if (PubSubHelper.isGatewayConfigured(dag)) {
        URI uri = PubSubHelper.getURI(dag);
        AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());
        Map<String, String> conversionMap = Maps.newHashMap();
        conversionMap.put(alias, WindowedTopCounter.FIELD_TYPE);
        String snapshotServerJSON = SchemaUtils.jarResourceFileToString(schemaFile);
        snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
        snapshotServer.setTableFieldToMapField(conversionMap);
        PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
        wsQuery.setUri(uri);
        snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);
        PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
        wsResult.setUri(uri);
        Operator.InputPort<String> queryResultPort = wsResult.input;
        dag.addStream("MapProvider", topCount, snapshotServer.input);
        dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
    } else {
        ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
        operator.setStringFormat(operatorName + ": %s");
        dag.addStream("MapProvider", topCount, operator.input);
    }
}

Example 4 with PubSubWebSocketAppDataQuery

use of org.apache.apex.malhar.lib.io.PubSubWebSocketAppDataQuery in project apex-malhar by apache.

the class ApplicationWithQuerySupport method populateDAG.

/**
 * Populates the DAG with operators and connecting streams
 *
 * @param dag The directed acyclic graph of operators to populate
 * @param conf The configuration
 */
@Override
public void populateDAG(DAG dag, Configuration conf) {
    // create operators
    LineReader lineReader = dag.addOperator("lineReader", new LineReader());
    WordReader wordReader = dag.addOperator("wordReader", new WordReader());
    WindowWordCount windowWordCount = dag.addOperator("windowWordCount", new WindowWordCount());
    FileWordCount fileWordCount = dag.addOperator("fileWordCount", new FileWordCount());
    WordCountWriter wcWriter = dag.addOperator("wcWriter", new WordCountWriter());
    ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
    console.setStringFormat("wordCount: %s");
    // create streams
    dag.addStream("lines", lineReader.output, wordReader.input);
    dag.addStream("control", lineReader.control, fileWordCount.control);
    dag.addStream("words", wordReader.output, windowWordCount.input);
    dag.addStream("windowWordCounts", windowWordCount.output, fileWordCount.input);
    dag.addStream("fileWordCounts", fileWordCount.fileOutput, wcWriter.input);
    if (PubSubHelper.isGatewayConfigured(dag)) {
        // add query support
        URI uri = PubSubHelper.getURI(dag);
        AppDataSnapshotServerMap snapshotServerFile = dag.addOperator("snapshotServerFile", new AppDataSnapshotServerMap());
        AppDataSnapshotServerMap snapshotServerGlobal = dag.addOperator("snapshotServerGlobal", new AppDataSnapshotServerMap());
        String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
        snapshotServerFile.setSnapshotSchemaJSON(snapshotServerJSON);
        snapshotServerGlobal.setSnapshotSchemaJSON(snapshotServerJSON);
        PubSubWebSocketAppDataQuery wsQueryFile = new PubSubWebSocketAppDataQuery();
        PubSubWebSocketAppDataQuery wsQueryGlobal = new PubSubWebSocketAppDataQuery();
        wsQueryFile.setUri(uri);
        wsQueryGlobal.setUri(uri);
        snapshotServerFile.setEmbeddableQueryInfoProvider(wsQueryFile);
        snapshotServerGlobal.setEmbeddableQueryInfoProvider(wsQueryGlobal);
        PubSubWebSocketAppDataResult wsResultFile = dag.addOperator("wsResultFile", new PubSubWebSocketAppDataResult());
        PubSubWebSocketAppDataResult wsResultGlobal = dag.addOperator("wsResultGlobal", new PubSubWebSocketAppDataResult());
        wsResultFile.setUri(uri);
        wsResultGlobal.setUri(uri);
        Operator.InputPort<String> queryResultFilePort = wsResultFile.input;
        Operator.InputPort<String> queryResultGlobalPort = wsResultGlobal.input;
        dag.addStream("WordCountsFile", fileWordCount.outputPerFile, snapshotServerFile.input, console.input);
        dag.addStream("WordCountsGlobal", fileWordCount.outputGlobal, snapshotServerGlobal.input);
        dag.addStream("ResultFile", snapshotServerFile.queryResult, queryResultFilePort);
        dag.addStream("ResultGlobal", snapshotServerGlobal.queryResult, queryResultGlobalPort);
    } else {
        // throw new RuntimeException("Error: No GATEWAY_CONNECT_ADDRESS");
        dag.addStream("WordCounts", fileWordCount.outputPerFile, console.input);
    }
    LOG.info("done with populateDAG, isDebugEnabled = " + LOG.isDebugEnabled());
    LOG.info("Returning from populateDAG");
}