Examples with KeyValPair org.apache.apex.malhar.lib.util.KeyValPair used on opensource projects

Example 31 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class WordCountWithStreamAPI method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration configuration) {
    WCInput wcInput = new WCInput();
    ApexStream<String> stream = StreamFactory.fromInput(wcInput, wcInput.output).flatMap(new Function.FlatMapFunction<String, String>() {

        @Override
        public Iterable<String> f(String input) {
            return Arrays.asList(input.split("[\\p{Punct}\\s]+"));
        }
    });
    stream.print();
    stream.window(new WindowOption.GlobalWindow(), new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(1000)).accumulatingFiredPanes()).countByKey(new Function.ToKeyValue<String, String, Long>() {

        @Override
        public Tuple<KeyValPair<String, Long>> f(String input) {
            return new Tuple.PlainTuple(new KeyValPair<>(input, 1L));
        }
    }).print();
    stream.populateDag(dag);
}

Example 32 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class MinKeyValTest method testSchemaNodeProcessing.

/**
 * Test operator logic emits correct results for each schema.
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public void testSchemaNodeProcessing(MinKeyVal oper, String type) {
    CountAndLastTupleTestSink minSink = new CountAndLastTupleTestSink();
    oper.min.setSink(minSink);
    oper.beginWindow(0);
    int numtuples = 10000;
    if (type.equals("integer")) {
        for (int i = numtuples; i > 0; i--) {
            oper.data.process(new KeyValPair("a", new Integer(i)));
        }
    } else if (type.equals("double")) {
        for (int i = numtuples; i > 0; i--) {
            oper.data.process(new KeyValPair("a", (double) i));
        }
    } else if (type.equals("long")) {
        for (int i = numtuples; i > 0; i--) {
            oper.data.process(new KeyValPair("a", (long) i));
        }
    } else if (type.equals("short")) {
        for (short j = 1000; j > 0; j--) {
            // cannot cross 64K
            oper.data.process(new KeyValPair("a", j));
        }
    } else if (type.equals("float")) {
        for (int i = numtuples; i > 0; i--) {
            oper.data.process(new KeyValPair("a", (float) i));
        }
    }
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 1, minSink.count);
    Number val = ((KeyValPair<String, Number>) minSink.tuple).getValue().intValue();
    if (type.equals("short")) {
        Assert.assertEquals("emitted min value was ", 1, val);
    } else {
        Assert.assertEquals("emitted min value was ", 1, val);
    }
}

Example 33 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class SumKeyValTest method testNodeProcessing.

/**
 * Test operator logic emits correct results.
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testNodeProcessing() {
    SumKeyVal<String, Double> oper = new SumKeyVal<String, Double>();
    oper.setType(Double.class);
    CollectorTestSink sumSink = new CollectorTestSink();
    oper.sum.setSink(sumSink);
    // 
    oper.beginWindow(0);
    oper.data.process(new KeyValPair("a", 2.0));
    oper.data.process(new KeyValPair("b", 20.0));
    oper.data.process(new KeyValPair("c", 1000.0));
    oper.data.process(new KeyValPair("a", 1.0));
    oper.data.process(new KeyValPair("a", 10.0));
    oper.data.process(new KeyValPair("b", 5.0));
    oper.data.process(new KeyValPair("d", 55.0));
    oper.data.process(new KeyValPair("b", 12.0));
    oper.data.process(new KeyValPair("d", 22.0));
    oper.data.process(new KeyValPair("d", 14.2));
    oper.data.process(new KeyValPair("d", 46.0));
    oper.data.process(new KeyValPair("e", 2.0));
    oper.data.process(new KeyValPair("a", 23.0));
    oper.data.process(new KeyValPair("d", 4.0));
    // 
    oper.endWindow();
    // payload should be 1 bag of tuples with keys "a", "b", "c", "d", "e"
    Assert.assertEquals("number emitted tuples", 5, sumSink.collectedTuples.size());
    for (Object o : sumSink.collectedTuples) {
        KeyValPair<String, Double> e = (KeyValPair<String, Double>) o;
        Double val = (Double) e.getValue();
        if (e.getKey().equals("a")) {
            Assert.assertEquals("emitted value for 'a' was ", new Double(36), val);
        } else if (e.getKey().equals("b")) {
            Assert.assertEquals("emitted tuple for 'b' was ", new Double(37), val);
        } else if (e.getKey().equals("c")) {
            Assert.assertEquals("emitted tuple for 'c' was ", new Double(1000), val);
        } else if (e.getKey().equals("d")) {
            Assert.assertEquals("emitted tuple for 'd' was ", new Double(141.2), val);
        } else if (e.getKey().equals("e")) {
            Assert.assertEquals("emitted tuple for 'e' was ", new Double(2), val);
        }
    }
}

Example 34 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class MultiWindowSumKeyValTest method testNodeProcessing.

/**
 * Test functional logic
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testNodeProcessing() throws InterruptedException {
    MultiWindowSumKeyVal<String, Integer> oper = new MultiWindowSumKeyVal<String, Integer>();
    CollectorTestSink swinSink = new CollectorTestSink();
    oper.sum.setSink(swinSink);
    oper.beginWindow(0);
    KeyValPair<String, Integer> low = new KeyValPair<String, Integer>("a", 3);
    oper.data.process(low);
    KeyValPair<String, Integer> high = new KeyValPair<String, Integer>("a", 11);
    oper.data.process(high);
    oper.endWindow();
    oper.beginWindow(1);
    low = new KeyValPair<String, Integer>("a", 1);
    oper.data.process(low);
    high = new KeyValPair<String, Integer>("a", 9);
    oper.data.process(high);
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 1, swinSink.collectedTuples.size());
    for (Object o : swinSink.collectedTuples) {
        log.debug(o.toString());
    }
}

Example 35 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class ConsolidatorKeyValTest method testNodeProcessing.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testNodeProcessing() throws Exception {
    ConsolidatorKeyVal<String, Integer, Double, Integer, Integer, Integer> oper = new ConsolidatorKeyVal<String, Integer, Double, Integer, Integer, Integer>();
    CollectorTestSink cSink = new CollectorTestSink();
    oper.out.setSink(cSink);
    oper.beginWindow(0);
    KeyValPair<String, Integer> m1 = new KeyValPair<String, Integer>("a", 1);
    oper.in1.process(m1);
    KeyValPair<String, Double> m2 = new KeyValPair<String, Double>("a", 1.0);
    oper.in2.process(m2);
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 1, cSink.collectedTuples.size());
    HashMap<String, ArrayList<Object>> map = (HashMap<String, ArrayList<Object>>) cSink.collectedTuples.get(0);
    Assert.assertEquals("size of sink map", 1, map.size());
}