Examples with KeyValPair org.apache.apex.malhar.lib.util.KeyValPair used on opensource projects

Example 6 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class TrafficRoutesTest method TrafficRoutesTest.

@Test
public void TrafficRoutesTest() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.set("dt.application.TrafficRoutes.operator.console.silent", "true");
    lma.prepareDAG(new TrafficRoutes(), conf);
    LocalMode.Controller lc = lma.getController();
    ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return TrafficRoutes.InfoGen.getTupleCount() >= 100;
        }
    });
    lc.run(60000);
    Assert.assertTrue(!TrafficRoutes.Collector.getResult().isEmpty());
    for (Map.Entry<KeyValPair<Long, String>, KeyValPair<Double, Boolean>> entry : TrafficRoutes.Collector.getResult().entrySet()) {
        Assert.assertTrue(entry.getValue().getKey() <= 75);
        Assert.assertTrue(entry.getValue().getKey() >= 55);
        Assert.assertTrue(entry.getKey().getValue().equals("SDRoute1") || entry.getKey().getValue().equals("SDRoute2"));
    }
}

Example 7 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class MaxKeyValTest method testSchemaNodeProcessing.

/**
 * Test operator logic emits correct results for each schema.
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testSchemaNodeProcessing(MaxKeyVal oper, String type) {
    CountAndLastTupleTestSink maxSink = new CountAndLastTupleTestSink();
    oper.max.setSink(maxSink);
    oper.beginWindow(0);
    int numtuples = 10000;
    if (type.equals("integer")) {
        for (int i = 0; i < numtuples; i++) {
            oper.data.process(new KeyValPair("a", i));
        }
    } else if (type.equals("double")) {
        for (int i = 0; i < numtuples; i++) {
            oper.data.process(new KeyValPair("a", (double) i));
        }
    } else if (type.equals("long")) {
        for (int i = 0; i < numtuples; i++) {
            oper.data.process(new KeyValPair("a", (long) i));
        }
    } else if (type.equals("short")) {
        // cannot cross 64K
        int count = numtuples / 1000;
        for (short j = 0; j < count; j++) {
            oper.data.process(new KeyValPair("a", j));
        }
    } else if (type.equals("float")) {
        for (int i = 0; i < numtuples; i++) {
            oper.data.process(new KeyValPair("a", (float) i));
        }
    }
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 1, maxSink.count);
    Number val = ((KeyValPair<String, Number>) maxSink.tuple).getValue().intValue();
    if (type.equals("short")) {
        Assert.assertEquals("emitted max value was ", (new Double(numtuples / 1000 - 1)).intValue(), val);
    } else {
        Assert.assertEquals("emitted max value was ", (new Double(numtuples - 1)).intValue(), val);
    }
}

Example 8 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class MultiWindowDimensionAggregation method endWindow.

@Override
public void endWindow() {
    int totalWindowsOccupied = cacheOject.size();
    for (Map.Entry<String, Map<String, KeyValPair<MutableDouble, Integer>>> e : outputMap.entrySet()) {
        for (Map.Entry<String, KeyValPair<MutableDouble, Integer>> dimensionValObj : e.getValue().entrySet()) {
            Map<String, DimensionObject<String>> outputData = new HashMap<String, DimensionObject<String>>();
            KeyValPair<MutableDouble, Integer> keyVal = dimensionValObj.getValue();
            if (operationType == AggregateOperation.SUM) {
                outputData.put(e.getKey(), new DimensionObject<String>(keyVal.getKey(), dimensionValObj.getKey()));
            } else if (operationType == AggregateOperation.AVERAGE) {
                if (keyVal.getValue() != 0) {
                    double totalCount = ((double) (totalWindowsOccupied * applicationWindowSize)) / 1000;
                    outputData.put(e.getKey(), new DimensionObject<String>(new MutableDouble(keyVal.getKey().doubleValue() / totalCount), dimensionValObj.getKey()));
                }
            }
            if (!outputData.isEmpty()) {
                output.emit(outputData);
            }
        }
    }
    currentWindow = (currentWindow + 1) % windowSize;
}

Example 9 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class AbstractPojoJoin method getAllCombo.

/**
 * This function takes the required join on the 2 input streams for matching keys
 * and allows the derived classes to implement the logic in case of non matching keys.
 *
 * It is designed such that for outer joins it will always assume that it is
 * a left outer join and hence it considers right stream as left in case of
 * right outer join keeping the code and logic the same.
 *
 * For each key in the left stream a corresponding key is searched in the right stream
 * if a match is found then the all the objects with that key are added to Output list,
 * also that key is removed from right stream as it will no longer be required in any join
 * scenario.If a match is not found then it relies on derived class implementation to handle it.
 *
 * @param accu which is the main accumulation data structure
 * @return List of objects got after joining the streams
 */
private List<Object> getAllCombo(List<Multimap<List<Object>, Object>> accu) {
    List<Object> result = new ArrayList<>();
    int leftStreamIndex = getLeftStreamIndex();
    Multimap<List<Object>, Object> leftStream = accu.get(leftStreamIndex);
    Multimap<List<Object>, Object> rightStream = ArrayListMultimap.create(accu.get((leftStreamIndex + 1) % 2));
    Map<String, PojoUtils.Getter> leftGettersStream = leftStreamIndex == 0 ? gettersStream1 : gettersStream2;
    Map<String, PojoUtils.Getter> rightGettersStream = leftStreamIndex == 1 ? gettersStream1 : gettersStream2;
    for (List lMap : leftStream.keySet()) {
        Collection<Object> left = leftStream.get(lMap);
        if (rightStream.containsKey(lMap)) {
            Collection<Object> right = rightStream.get(lMap);
            Object o;
            try {
                o = outClass.newInstance();
            } catch (Throwable e) {
                throw Throwables.propagate(e);
            }
            for (Object lObj : left) {
                for (Object rObj : right) {
                    if (outputToInputMap != null) {
                        for (Map.Entry<String, KeyValPair<STREAM, String>> entry : outputToInputMap.entrySet()) {
                            KeyValPair<STREAM, String> kv = entry.getValue();
                            Object reqObject;
                            Map<String, PojoUtils.Getter> reqStream;
                            if (kv.getKey() == LEFT) {
                                reqObject = leftStreamIndex == 0 ? lObj : rObj;
                                reqStream = leftStreamIndex == 0 ? leftGettersStream : rightGettersStream;
                            } else {
                                reqObject = leftStreamIndex == 0 ? rObj : lObj;
                                reqStream = leftStreamIndex == 0 ? rightGettersStream : leftGettersStream;
                            }
                            setters.get(entry.getKey()).set(o, reqStream.get(entry.getValue().getValue()).get(reqObject));
                        }
                    } else {
                        setObjectForResult(leftGettersStream, lObj, o);
                        setObjectForResult(rightGettersStream, rObj, o);
                    }
                }
                result.add(o);
            }
            rightStream.removeAll(lMap);
        } else {
            addNonMatchingResult(left, leftGettersStream, result);
        }
    }
    addNonMatchingRightStream(rightStream, rightGettersStream, result);
    return result;
}

Example 10 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class CountKeyValTest method testNodeProcessing.

/**
 * Test operator logic emits correct results.
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testNodeProcessing() {
    CountKeyVal<String, Double> oper = new CountKeyVal<String, Double>();
    CollectorTestSink countSink = new CollectorTestSink();
    oper.count.setSink(countSink);
    // 
    oper.beginWindow(0);
    oper.data.process(new KeyValPair("a", 2.0));
    oper.data.process(new KeyValPair("b", 20.0));
    oper.data.process(new KeyValPair("c", 1000.0));
    oper.data.process(new KeyValPair("a", 1.0));
    oper.data.process(new KeyValPair("a", 10.0));
    oper.data.process(new KeyValPair("b", 5.0));
    oper.data.process(new KeyValPair("d", 55.0));
    oper.data.process(new KeyValPair("b", 12.0));
    oper.data.process(new KeyValPair("d", 22.0));
    oper.data.process(new KeyValPair("d", 14.2));
    oper.data.process(new KeyValPair("d", 46.0));
    oper.data.process(new KeyValPair("e", 2.0));
    oper.data.process(new KeyValPair("a", 23.0));
    oper.data.process(new KeyValPair("d", 4.0));
    // 
    oper.endWindow();
    // payload should be 1 bag of tuples with keys "a", "b", "c", "d", "e"
    Assert.assertEquals("number emitted tuples", 5, countSink.collectedTuples.size());
    for (Object o : countSink.collectedTuples) {
        KeyValPair<String, Integer> e = (KeyValPair<String, Integer>) o;
        Integer val = (Integer) e.getValue();
        if (e.getKey().equals("a")) {
            Assert.assertEquals("emitted value for 'a' was ", 4, val.intValue());
        } else if (e.getKey().equals("b")) {
            Assert.assertEquals("emitted tuple for 'b' was ", 3, val.intValue());
        } else if (e.getKey().equals("c")) {
            Assert.assertEquals("emitted tuple for 'c' was ", 1, val.intValue());
        } else if (e.getKey().equals("d")) {
            Assert.assertEquals("emitted tuple for 'd' was ", 5, val.intValue());
        } else if (e.getKey().equals("e")) {
            Assert.assertEquals("emitted tuple for 'e' was ", 1, val.intValue());
        }
    }
}