Examples with KeyValPair org.apache.apex.malhar.lib.util.KeyValPair used on opensource projects

Example 1 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class Application method populateDAG.

@SuppressWarnings("unchecked")
@Override
public void populateDAG(DAG dag, Configuration conf) {
    try {
        URI duri = PubSubHelper.getURIWithDefault(dag, "localhost:9090");
        PubSubWebSocketInputOperator userTxWsInput = getPubSubWebSocketInputOperator("userTxInput", dag, duri, "examples.app.frauddetect.submitTransaction");
        PubSubWebSocketOutputOperator ccUserAlertWsOutput = getPubSubWebSocketOutputOperator("ccUserAlertQueryOutput", dag, duri, "examples.app.frauddetect.fraudAlert");
        PubSubWebSocketOutputOperator avgUserAlertwsOutput = getPubSubWebSocketOutputOperator("avgUserAlertQueryOutput", dag, duri, "examples.app.frauddetect.fraudAlert");
        PubSubWebSocketOutputOperator binUserAlertwsOutput = getPubSubWebSocketOutputOperator("binUserAlertOutput", dag, duri, "examples.app.frauddetect.fraudAlert");
        PubSubWebSocketOutputOperator txSummaryWsOutput = getPubSubWebSocketOutputOperator("txSummaryWsOutput", dag, duri, "examples.app.frauddetect.txSummary");
        SlidingWindowSumKeyVal<KeyValPair<MerchantKey, String>, Integer> smsOperator = dag.addOperator("movingSum", SlidingWindowSumKeyVal.class);
        MerchantTransactionGenerator txReceiver = dag.addOperator("txReceiver", MerchantTransactionGenerator.class);
        MerchantTransactionInputHandler txInputHandler = dag.addOperator("txInputHandler", new MerchantTransactionInputHandler());
        BankIdNumberSamplerOperator binSampler = dag.addOperator("bankInfoFraudDetector", BankIdNumberSamplerOperator.class);
        MerchantTransactionBucketOperator txBucketOperator = dag.addOperator("txFilter", MerchantTransactionBucketOperator.class);
        RangeKeyVal rangeOperator = dag.addOperator("rangePerMerchant", new RangeKeyVal<MerchantKey, Long>());
        SimpleMovingAverage<MerchantKey, Long> smaOperator = dag.addOperator("smaPerMerchant", SimpleMovingAverage.class);
        TransactionStatsAggregator txStatsAggregator = dag.addOperator("txStatsAggregator", TransactionStatsAggregator.class);
        AverageAlertingOperator avgAlertingOperator = dag.addOperator("avgAlerter", AverageAlertingOperator.class);
        CreditCardAmountSamplerOperator ccSamplerOperator = dag.addOperator("amountFraudDetector", CreditCardAmountSamplerOperator.class);
        HdfsStringOutputOperator hdfsOutputOperator = getHdfsOutputOperator("hdfsOutput", dag, "fraud");
        MongoDBOutputOperator mongoTxStatsOperator = dag.addOperator("mongoTxStatsOutput", MongoDBOutputOperator.class);
        MongoDBOutputOperator mongoBinAlertsOperator = dag.addOperator("mongoBinAlertsOutput", MongoDBOutputOperator.class);
        MongoDBOutputOperator mongoCcAlertsOperator = dag.addOperator("mongoCcAlertsOutput", MongoDBOutputOperator.class);
        MongoDBOutputOperator mongoAvgAlertsOperator = dag.addOperator("mongoAvgAlertsOutput", MongoDBOutputOperator.class);
        dag.addStream("userTxStream", userTxWsInput.outputPort, txInputHandler.userTxInputPort);
        dag.addStream("transactions", txReceiver.txOutputPort, txBucketOperator.inputPort).setLocality(DAG.Locality.CONTAINER_LOCAL);
        // dump all tx into Hdfs
        dag.addStream("txData", txReceiver.txDataOutputPort, hdfsOutputOperator.input);
        dag.addStream("userTransactions", txInputHandler.txOutputPort, txBucketOperator.txUserInputPort);
        dag.addStream("bankInfoData", txBucketOperator.binCountOutputPort, smsOperator.data);
        dag.addStream("bankInfoCount", smsOperator.integerSum, binSampler.txCountInputPort);
        dag.addStream("filteredTransactions", txBucketOperator.txOutputPort, rangeOperator.data, smaOperator.data, avgAlertingOperator.txInputPort);
        KeyPartitionCodec<MerchantKey, Long> txCodec = new KeyPartitionCodec<MerchantKey, Long>();
        dag.setInputPortAttribute(rangeOperator.data, Context.PortContext.STREAM_CODEC, txCodec);
        dag.setInputPortAttribute(smaOperator.data, Context.PortContext.STREAM_CODEC, txCodec);
        dag.setInputPortAttribute(avgAlertingOperator.txInputPort, Context.PortContext.STREAM_CODEC, txCodec);
        dag.addStream("creditCardData", txBucketOperator.ccAlertOutputPort, ccSamplerOperator.inputPort);
        dag.addStream("txnSummaryData", txBucketOperator.summaryTxnOutputPort, txSummaryWsOutput.input);
        dag.addStream("smaAlerts", smaOperator.doubleSMA, avgAlertingOperator.smaInputPort);
        dag.addStream("binAlerts", binSampler.countAlertOutputPort, mongoBinAlertsOperator.inputPort);
        dag.addStream("binAlertsNotification", binSampler.countAlertNotificationPort, binUserAlertwsOutput.input);
        dag.addStream("rangeData", rangeOperator.range, txStatsAggregator.rangeInputPort);
        dag.addStream("smaData", smaOperator.longSMA, txStatsAggregator.smaInputPort);
        dag.addStream("txStatsOutput", txStatsAggregator.txDataOutputPort, mongoTxStatsOperator.inputPort);
        dag.addStream("avgAlerts", avgAlertingOperator.avgAlertOutputPort, mongoAvgAlertsOperator.inputPort);
        dag.addStream("avgAlertsNotification", avgAlertingOperator.avgAlertNotificationPort, avgUserAlertwsOutput.input);
        dag.addStream("ccAlerts", ccSamplerOperator.ccAlertOutputPort, mongoCcAlertsOperator.inputPort);
        dag.addStream("ccAlertsNotification", ccSamplerOperator.ccAlertNotificationPort, ccUserAlertWsOutput.input);
    } catch (Exception exc) {
        DTThrowable.rethrow(exc);
    }
}

Example 2 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class Application method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration conf) {
    RandomKeyValGenerator randGen = dag.addOperator("RandomGenerator", new RandomKeyValGenerator());
    UniqueValueCount<Integer> valCount = dag.addOperator("UniqueCounter", new UniqueValueCount<Integer>());
    ConsoleOutputOperator consOut = dag.addOperator("Console", new ConsoleOutputOperator());
    StreamDuplicater<KeyValPair<Integer, Integer>> dup = dag.addOperator("Duplicator", new StreamDuplicater<KeyValPair<Integer, Integer>>());
    CountVerifier verifier = dag.addOperator("Verifier", new CountVerifier());
    ConsoleOutputOperator successOutput = dag.addOperator("Success", new ConsoleOutputOperator());
    successOutput.setStringFormat("Success %d");
    ConsoleOutputOperator failureOutput = dag.addOperator("Failure", new ConsoleOutputOperator());
    failureOutput.setStringFormat("Failure %d");
    Counter successcounter = dag.addOperator("SuccessCounter", new Counter());
    Counter failurecounter = dag.addOperator("FailureCounter", new Counter());
    dag.addStream("Events", randGen.outport, valCount.input);
    dag.addStream("Duplicates", valCount.output, dup.data);
    dag.addStream("Unverified", dup.out1, verifier.recIn);
    dag.addStream("EventCount", randGen.verport, verifier.trueIn);
    dag.addStream("Verified", verifier.successPort, successcounter.input);
    dag.addStream("Failed", verifier.failurePort, failurecounter.input);
    dag.addStream("SuccessCount", successcounter.output, successOutput.input);
    dag.addStream("FailedCount", failurecounter.output, failureOutput.input);
    dag.addStream("Output", dup.out2, consOut.input);
}

Example 3 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class RedisInputOperatorTest method testIntputOperator.

@Test
public void testIntputOperator() throws IOException {
    this.operatorStore = new RedisStore();
    this.testStore = new RedisStore();
    testStore.connect();
    ScanParams params = new ScanParams();
    params.count(1);
    testStore.put("test_abc", "789");
    testStore.put("test_def", "456");
    testStore.put("test_ghi", "123");
    try {
        LocalMode lma = LocalMode.newInstance();
        DAG dag = lma.getDAG();
        RedisKeyValueInputOperator inputOperator = dag.addOperator("input", new RedisKeyValueInputOperator());
        final CollectorModule collector = dag.addOperator("collector", new CollectorModule());
        inputOperator.setStore(operatorStore);
        dag.addStream("stream", inputOperator.outputPort, collector.inputPort);
        final LocalMode.Controller lc = lma.getController();
        new Thread("LocalClusterController") {

            @Override
            public void run() {
                long startTms = System.currentTimeMillis();
                long timeout = 50000L;
                try {
                    Thread.sleep(1000);
                    while (System.currentTimeMillis() - startTms < timeout) {
                        if (CollectorModule.resultMap.size() < 3) {
                            Thread.sleep(10);
                        } else {
                            break;
                        }
                    }
                } catch (InterruptedException ex) {
                // 
                }
                lc.shutdown();
            }
        }.start();
        lc.run();
        Assert.assertTrue(CollectorModule.resultMap.contains(new KeyValPair<String, String>("test_abc", "789")));
        Assert.assertTrue(CollectorModule.resultMap.contains(new KeyValPair<String, String>("test_def", "456")));
        Assert.assertTrue(CollectorModule.resultMap.contains(new KeyValPair<String, String>("test_ghi", "123")));
    } finally {
        for (KeyValPair<String, String> entry : CollectorModule.resultMap) {
            testStore.remove(entry.getKey());
        }
        testStore.disconnect();
    }
}

Example 4 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class RedisInputOperatorTest method testRecoveryAndIdempotency.

@Test
public void testRecoveryAndIdempotency() throws Exception {
    this.operatorStore = new RedisStore();
    this.testStore = new RedisStore();
    testStore.connect();
    ScanParams params = new ScanParams();
    params.count(1);
    testStore.put("test_abc", "789");
    testStore.put("test_def", "456");
    testStore.put("test_ghi", "123");
    RedisKeyValueInputOperator operator = new RedisKeyValueInputOperator();
    operator.setWindowDataManager(new FSWindowDataManager());
    operator.setStore(operatorStore);
    operator.setScanCount(1);
    Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
    CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
    operator.outputPort.setSink(sink);
    OperatorContext context = mockOperatorContext(1, attributeMap);
    try {
        operator.setup(context);
        operator.beginWindow(1);
        operator.emitTuples();
        operator.endWindow();
        int numberOfMessagesInWindow1 = sink.collectedTuples.size();
        sink.collectedTuples.clear();
        operator.beginWindow(2);
        operator.emitTuples();
        operator.endWindow();
        int numberOfMessagesInWindow2 = sink.collectedTuples.size();
        sink.collectedTuples.clear();
        // failure and then re-deployment of operator
        // Re-instantiating to reset values
        operator = new RedisKeyValueInputOperator();
        operator.setWindowDataManager(new FSWindowDataManager());
        operator.setStore(operatorStore);
        operator.setScanCount(1);
        operator.outputPort.setSink(sink);
        operator.setup(context);
        Assert.assertEquals("largest recovery window", 2, operator.getWindowDataManager().getLargestCompletedWindow());
        operator.beginWindow(1);
        operator.emitTuples();
        operator.emitTuples();
        operator.endWindow();
        Assert.assertEquals("num of messages in window 1", numberOfMessagesInWindow1, sink.collectedTuples.size());
        sink.collectedTuples.clear();
        operator.beginWindow(2);
        operator.emitTuples();
        operator.endWindow();
        Assert.assertEquals("num of messages in window 2", numberOfMessagesInWindow2, sink.collectedTuples.size());
    } finally {
        for (Object e : sink.collectedTuples) {
            KeyValPair<String, String> entry = (KeyValPair<String, String>) e;
            testStore.remove(entry.getKey());
        }
        sink.collectedTuples.clear();
        operator.getWindowDataManager().committed(5);
        operator.teardown();
    }
}

Example 5 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class RedisPOJOOperatorTest method testOutputOperator.

@Test
public void testOutputOperator() throws IOException {
    this.operatorStore = new RedisStore();
    operatorStore.connect();
    String appId = "test_appid";
    int operatorId = 0;
    operatorStore.removeCommittedWindowId(appId, operatorId);
    operatorStore.disconnect();
    RedisPOJOOutputOperator outputOperator = new RedisPOJOOutputOperator();
    ArrayList<FieldInfo> fields = new ArrayList<FieldInfo>();
    fields.add(new FieldInfo("column1", "intValue", SupportType.INTEGER));
    fields.add(new FieldInfo("column2", "getStringValue()", SupportType.STRING));
    outputOperator.setDataColumns(fields);
    try {
        com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap attributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
        attributes.put(DAG.APPLICATION_ID, appId);
        outputOperator.setStore(operatorStore);
        outputOperator.setup(mockOperatorContext(operatorId, attributes));
        outputOperator.beginWindow(101);
        KeyValPair<String, Object> keyVal = new KeyValPair<String, Object>("test_abc1", new TestClass(1, "abc"));
        outputOperator.input.process(keyVal);
        outputOperator.endWindow();
        outputOperator.teardown();
        operatorStore.connect();
        Map<String, String> out = operatorStore.getMap("test_abc1");
        Assert.assertEquals("1", out.get("column1"));
        Assert.assertEquals("abc", out.get("column2"));
    } finally {
        operatorStore.remove("test_abc1");
        operatorStore.disconnect();
    }
}