Examples with KeyValPair org.apache.apex.malhar.lib.util.KeyValPair used on opensource projects

Example 26 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class UniqueKeyValCountExample method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration entries) {
    /* Generate random key-value pairs */
    RandomDataGenerator randGen = dag.addOperator("randomgen", new RandomDataGenerator());
    /* Initialize with three partition to start with */
    UniqueCounter<KeyValPair<String, Object>> uniqCount = dag.addOperator("uniqevalue", new UniqueCounter<KeyValPair<String, Object>>());
    MapToKeyHashValuePairConverter<KeyValPair<String, Object>, Integer> converter = dag.addOperator("converter", new MapToKeyHashValuePairConverter());
    uniqCount.setCumulative(false);
    dag.setAttribute(randGen, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<UniqueCounter<KeyValPair<String, Object>>>(3));
    ConsoleOutputOperator output = dag.addOperator("output", new ConsoleOutputOperator());
    dag.addStream("datain", randGen.outPort, uniqCount.data);
    dag.addStream("convert", uniqCount.count, converter.input).setLocality(Locality.THREAD_LOCAL);
    dag.addStream("consoutput", converter.output, output.input);
}

Example 27 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class CalculatorOperatorTest method testMax.

public void testMax(CalculatorOperator oper) {
    CollectorTestSink sortSink = new CollectorTestSink();
    oper.maxOutputPort.setSink(sortSink);
    Calendar calendar = Calendar.getInstance();
    Date date = calendar.getTime();
    String timeKey = minuteDateFormat.format(date);
    String day = calendar.get(Calendar.DAY_OF_MONTH) + "";
    Integer vs = new Integer(1);
    MachineKey mk = new MachineKey(timeKey, day, vs, vs, vs, vs, vs, vs, vs);
    oper.beginWindow(0);
    MachineInfo info = new MachineInfo(mk, 1, 1, 1);
    oper.dataPort.process(info);
    info.setCpu(2);
    oper.dataPort.process(info);
    info.setCpu(3);
    oper.dataPort.process(info);
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 1, sortSink.collectedTuples.size());
    for (Object o : sortSink.collectedTuples) {
        LOG.debug(o.toString());
        KeyValPair<TimeBucketKey, Map<ResourceType, Double>> keyValPair = (KeyValPair<TimeBucketKey, Map<ResourceType, Double>>) o;
        Assert.assertEquals("emitted value for 'cpu' was ", 3, keyValPair.getValue().get(ResourceType.CPU), 0);
        Assert.assertEquals("emitted value for 'hdd' was ", 1, keyValPair.getValue().get(ResourceType.HDD), 0);
        Assert.assertEquals("emitted value for 'ram' was ", 1, keyValPair.getValue().get(ResourceType.RAM), 0);
    }
    LOG.debug("Done max testing\n");
}

Example 28 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class ChangeAlertKeyValTest method testNodeProcessingSchema.

@SuppressWarnings({ "rawtypes", "unchecked" })
public <V extends Number> void testNodeProcessingSchema(ChangeAlertKeyVal<String, V> oper) {
    CollectorTestSink alertSink = new CollectorTestSink();
    oper.alert.setSink(alertSink);
    oper.setPercentThreshold(5);
    oper.beginWindow(0);
    oper.data.process(new KeyValPair<String, V>("a", oper.getValue(200)));
    oper.data.process(new KeyValPair<String, V>("b", oper.getValue(10)));
    oper.data.process(new KeyValPair<String, V>("c", oper.getValue(100)));
    oper.data.process(new KeyValPair<String, V>("a", oper.getValue(203)));
    oper.data.process(new KeyValPair<String, V>("b", oper.getValue(12)));
    oper.data.process(new KeyValPair<String, V>("c", oper.getValue(101)));
    oper.data.process(new KeyValPair<String, V>("a", oper.getValue(210)));
    oper.data.process(new KeyValPair<String, V>("b", oper.getValue(12)));
    oper.data.process(new KeyValPair<String, V>("c", oper.getValue(102)));
    oper.data.process(new KeyValPair<String, V>("a", oper.getValue(231)));
    oper.data.process(new KeyValPair<String, V>("b", oper.getValue(18)));
    oper.data.process(new KeyValPair<String, V>("c", oper.getValue(103)));
    oper.endWindow();
    // One for a, Two for b
    Assert.assertEquals("number emitted tuples", 3, alertSink.collectedTuples.size());
    double aval = 0;
    double bval = 0;
    log.debug("\nLogging tuples");
    for (Object o : alertSink.collectedTuples) {
        KeyValPair<String, KeyValPair<Number, Double>> map = (KeyValPair<String, KeyValPair<Number, Double>>) o;
        log.debug(o.toString());
        if (map.getKey().equals("a")) {
            KeyValPair<Number, Double> vmap = map.getValue();
            if (vmap != null) {
                aval += vmap.getValue().doubleValue();
            }
        } else {
            KeyValPair<Number, Double> vmap = map.getValue();
            if (vmap != null) {
                bval += vmap.getValue().doubleValue();
            }
        }
    }
    Assert.assertEquals("change in a", 10.0, aval, 0);
    Assert.assertEquals("change in a", 70.0, bval, 0);
}

Example 29 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class ChangeKeyValTest method testNodeProcessingSchema.

/**
 * @param oper
 *          key/value pair for comparison.
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public <V extends Number> void testNodeProcessingSchema(ChangeKeyVal<String, V> oper) {
    CollectorTestSink changeSink = new CollectorTestSink();
    CollectorTestSink percentSink = new CollectorTestSink();
    oper.change.setSink(changeSink);
    oper.percent.setSink(percentSink);
    oper.beginWindow(0);
    oper.base.process(new KeyValPair<String, V>("a", oper.getValue(2)));
    oper.base.process(new KeyValPair<String, V>("b", oper.getValue(10)));
    oper.base.process(new KeyValPair<String, V>("c", oper.getValue(100)));
    oper.data.process(new KeyValPair<String, V>("a", oper.getValue(3)));
    oper.data.process(new KeyValPair<String, V>("b", oper.getValue(2)));
    oper.data.process(new KeyValPair<String, V>("c", oper.getValue(4)));
    oper.endWindow();
    // One for each key
    Assert.assertEquals("number emitted tuples", 3, changeSink.collectedTuples.size());
    Assert.assertEquals("number emitted tuples", 3, percentSink.collectedTuples.size());
    log.debug("\nLogging tuples");
    for (Object o : changeSink.collectedTuples) {
        KeyValPair<String, Number> kv = (KeyValPair<String, Number>) o;
        if (kv.getKey().equals("a")) {
            Assert.assertEquals("change in a ", 1.0, kv.getValue());
        }
        if (kv.getKey().equals("b")) {
            Assert.assertEquals("change in b ", -8.0, kv.getValue());
        }
        if (kv.getKey().equals("c")) {
            Assert.assertEquals("change in c ", -96.0, kv.getValue());
        }
    }
    for (Object o : percentSink.collectedTuples) {
        KeyValPair<String, Number> kv = (KeyValPair<String, Number>) o;
        if (kv.getKey().equals("a")) {
            Assert.assertEquals("change in a ", 50.0, kv.getValue());
        }
        if (kv.getKey().equals("b")) {
            Assert.assertEquals("change in b ", -80.0, kv.getValue());
        }
        if (kv.getKey().equals("c")) {
            Assert.assertEquals("change in c ", -96.0, kv.getValue());
        }
    }
}

Example 30 with KeyValPair

use of org.apache.apex.malhar.lib.util.KeyValPair in project apex-malhar by apache.

the class ApplicationWithStreamAPI method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration configuration) {
    String localFolder = "./src/test/resources/data";
    ApexStream<String> stream = StreamFactory.fromFolder(localFolder).flatMap(new Function.FlatMapFunction<String, String>() {

        @Override
        public Iterable<String> f(String input) {
            return Arrays.asList(input.split("[\\p{Punct}\\s]+"));
        }
    });
    stream.print(name("WordOutput"));
    stream.window(new WindowOption.GlobalWindow(), new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(1000)).accumulatingFiredPanes()).countByKey(new Function.ToKeyValue<String, String, Long>() {

        @Override
        public Tuple<KeyValPair<String, Long>> f(String input) {
            return new Tuple.PlainTuple(new KeyValPair<>(input, 1L));
        }
    }).print(name("WCOutput"));
    stream.populateDag(dag);
}