Examples with CollectorTestSink org.apache.apex.malhar.lib.testbench.CollectorTestSink used on opensource projects

Example 56 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class DimensionOperatorUnifierTest method testOperator.

@Test
@SuppressWarnings("unchecked")
public void testOperator() {
    DimensionOperatorUnifier unifier = new DimensionOperatorUnifier();
    CollectorTestSink sink = new CollectorTestSink();
    unifier.aggregationsOutput.setSink(sink);
    unifier.beginWindow(1);
    Map<String, DimensionObject<String>> tuple1 = new HashMap<String, DimensionObject<String>>();
    tuple1.put("m|201402121900|0|65537|131074|bytes.AVERAGE", new DimensionObject<String>(new MutableDouble(75), "a"));
    tuple1.put("m|201402121900|0|65537|131074|bytes.COUNT", new DimensionObject<String>(new MutableDouble(3.0), "a"));
    tuple1.put("m|201402121900|0|65537|131074|bytes.SUM", new DimensionObject<String>(new MutableDouble(225), "a"));
    Map<String, DimensionObject<String>> tuple2 = new HashMap<String, DimensionObject<String>>();
    tuple2.put("m|201402121900|0|65537|131074|bytes.AVERAGE", new DimensionObject<String>(new MutableDouble(50), "a"));
    tuple2.put("m|201402121900|0|65537|131074|bytes.COUNT", new DimensionObject<String>(new MutableDouble(2.0), "a"));
    tuple2.put("m|201402121900|0|65537|131074|bytes.SUM", new DimensionObject<String>(new MutableDouble(100), "a"));
    Map<String, DimensionObject<String>> tuple3 = new HashMap<String, DimensionObject<String>>();
    tuple3.put("m|201402121900|0|65537|131074|bytes.AVERAGE", new DimensionObject<String>(new MutableDouble(50), "z"));
    tuple3.put("m|201402121900|0|65537|131074|bytes.COUNT", new DimensionObject<String>(new MutableDouble(2.0), "z"));
    tuple3.put("m|201402121900|0|65537|131074|bytes.SUM", new DimensionObject<String>(new MutableDouble(100), "z"));
    Map<String, DimensionObject<String>> tuple4 = new HashMap<String, DimensionObject<String>>();
    tuple4.put("m|201402121900|0|65537|131075|bytes.AVERAGE", new DimensionObject<String>(new MutableDouble(14290.5), "b"));
    tuple4.put("m|201402121900|0|65537|131075|bytes.COUNT", new DimensionObject<String>(new MutableDouble(2.0), "b"));
    tuple4.put("m|201402121900|0|65537|131075|bytes.SUM", new DimensionObject<String>(new MutableDouble(28581.0), "b"));
    Map<String, DimensionObject<String>> tuple5 = new HashMap<String, DimensionObject<String>>();
    tuple5.put("m|201402121900|0|65537|131076|bytes.AVERAGE", new DimensionObject<String>(new MutableDouble(290.75), "c"));
    tuple5.put("m|201402121900|0|65537|131076|bytes.COUNT", new DimensionObject<String>(new MutableDouble(10.0), "c"));
    tuple5.put("m|201402121900|0|65537|131076|bytes.SUM", new DimensionObject<String>(new MutableDouble(8581.0), "c"));
    unifier.process(tuple1);
    unifier.process(tuple2);
    unifier.process(tuple3);
    unifier.process(tuple4);
    unifier.process(tuple5);
    unifier.endWindow();
    @SuppressWarnings("unchecked") List<Map<String, DimensionObject<String>>> tuples = sink.collectedTuples;
    Assert.assertEquals("Tuple Count", 4, tuples.size());
    for (Map<String, DimensionObject<String>> map : tuples) {
        for (Entry<String, DimensionObject<String>> entry : map.entrySet()) {
            String key = entry.getKey();
            DimensionObject<String> dimObj = entry.getValue();
            if (key.equals("m|201402121900|0|65537|131074|bytes.AVERAGE") && dimObj.getVal().equals("a")) {
                Assert.assertEquals("average for key " + key + " and dimension key " + "a", new MutableDouble(65), dimObj.getCount());
            }
            if (key.equals("m|201402121900|0|65537|131074|bytes.SUM") && dimObj.getVal().equals("z")) {
                Assert.assertEquals("sum for key " + key + " and dimension key " + "z", new MutableDouble(100), dimObj.getCount());
            }
            if (key.equals("m|201402121900|0|65537|131076|bytes.COUNT") && dimObj.getVal().equals("c")) {
                Assert.assertEquals("count for key " + key + " and dimension key " + "c", new MutableDouble(10), dimObj.getCount());
            }
        }
    }
}

Example 57 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class FilterOperatorTest method testOperator.

@Test
@SuppressWarnings("unchecked")
public void testOperator() {
    FilterOperator oper = new FilterOperator();
    LogstreamPropertyRegistry registry = new LogstreamPropertyRegistry();
    registry.bind(LogstreamUtil.LOG_TYPE, "apache");
    oper.setRegistry(registry);
    oper.setup(null);
    String filter1 = "a==\"1\"&&b==\"2\"&&c_info==\"abc\"";
    oper.addFilterCondition(new String[] { "type=apache", "a", "b", "c_info", "d", filter1 });
    String filter2 = "d==1";
    oper.addFilterCondition(new String[] { "type=apache", "d", filter2 });
    String filter3 = "a==\"1\"";
    oper.addFilterCondition(new String[] { "type=apache", "a", filter3 });
    String filter4 = "e==\"2\"";
    oper.addFilterCondition(new String[] { "type=apache", "e", filter4 });
    String filter5 = "response.equals(\"404\")";
    oper.addFilterCondition(new String[] { "type=apache", "response", filter5 });
    String filter6 = "default=true";
    oper.addFilterCondition(new String[] { "type=apache", filter6 });
    HashMap<String, Object> inMap = new HashMap<String, Object>();
    inMap.put(LogstreamUtil.LOG_TYPE, registry.getIndex(LogstreamUtil.LOG_TYPE, "apache"));
    inMap.put("a", "1");
    inMap.put("b", "2");
    inMap.put("c_info", "abc");
    inMap.put("d", 1);
    inMap.put("e", "3");
    inMap.put("response", "404");
    Set<Integer> expectedPassSet = new HashSet<Integer>();
    expectedPassSet.add(registry.getIndex(LogstreamUtil.FILTER, filter1));
    expectedPassSet.add(registry.getIndex(LogstreamUtil.FILTER, filter2));
    expectedPassSet.add(registry.getIndex(LogstreamUtil.FILTER, filter3));
    expectedPassSet.add(registry.getIndex(LogstreamUtil.FILTER, filter5));
    expectedPassSet.add(registry.getIndex(LogstreamUtil.FILTER, "apache_DEFAULT"));
    CollectorTestSink mapSink = new CollectorTestSink();
    oper.outputMap.setSink(mapSink);
    oper.beginWindow(0);
    oper.input.process(inMap);
    oper.endWindow();
    Assert.assertEquals("tuple count", 5, mapSink.collectedTuples.size());
    List<HashMap<String, Object>> tuples = mapSink.collectedTuples;
    Set<Integer> actualPassSet = new HashSet<Integer>();
    for (HashMap<String, Object> tuple : tuples) {
        Integer filter = (Integer) tuple.get(LogstreamUtil.FILTER);
        actualPassSet.add(filter);
    }
    Assert.assertEquals("Passed filters", expectedPassSet, actualPassSet);
}

Example 58 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class ElasticSearchPercolateTest method checkPercolateResponse.

/**
 */
private void checkPercolateResponse() {
    ElasticSearchPercolatorOperator oper = new ElasticSearchPercolatorOperator();
    oper.hostName = HOST_NAME;
    oper.port = PORT;
    oper.indexName = INDEX_NAME;
    oper.documentType = DOCUMENT_TYPE;
    oper.setup(null);
    String[] messages = { "{content:'This will match only with malhar'}", "{content:'This will match only with github'}", "{content:'This will match with both github and malhar'}", "{content:'This will not match with any of them'}" };
    String[][] matches = { { MALHAR_TOPIC }, { GITHUB_TOPIC }, { GITHUB_TOPIC, MALHAR_TOPIC }, {} };
    CollectorTestSink<PercolateResponse> sink = new CollectorTestSink<PercolateResponse>();
    oper.outputPort.setSink((CollectorTestSink) sink);
    for (String message : messages) {
        oper.inputPort.process(message);
    }
    int i = 0;
    for (PercolateResponse response : sink.collectedTuples) {
        List<String> matchIds = new ArrayList<String>();
        for (Match match : response.getMatches()) {
            matchIds.add(match.getId().toString());
        }
        Collections.sort(matchIds);
        Assert.assertArrayEquals(matchIds.toArray(), matches[i]);
        i++;
    }
}

Example 59 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class FileEnrichmentTest method testEnrichmentOperator.

@Test
public void testEnrichmentOperator() throws IOException, InterruptedException {
    URL origUrl = this.getClass().getResource("/productmapping.txt");
    URL fileUrl = new URL(this.getClass().getResource("/").toString() + "productmapping1.txt");
    FileUtils.deleteQuietly(new File(fileUrl.getPath()));
    FileUtils.copyFile(new File(origUrl.getPath()), new File(fileUrl.getPath()));
    MapEnricher oper = new MapEnricher();
    FSLoader store = new JsonFSLoader();
    store.setFileName(fileUrl.toString());
    oper.setLookupFields(Arrays.asList("productId"));
    oper.setIncludeFields(Arrays.asList("productCategory"));
    oper.setStore(store);
    oper.setup(null);
    /* File contains 6 entries, but operator one entry is duplicate,
     * so cache should contains only 5 entries after scanning input file.
     */
    // Assert.assertEquals("Number of mappings ", 7, oper.cache.size());
    CollectorTestSink<Map<String, Object>> sink = new CollectorTestSink<>();
    @SuppressWarnings({ "unchecked", "rawtypes" }) CollectorTestSink<Object> tmp = (CollectorTestSink) sink;
    oper.output.setSink(tmp);
    oper.activate(null);
    oper.beginWindow(0);
    Map<String, Object> tuple = Maps.newHashMap();
    tuple.put("productId", 3);
    tuple.put("channelId", 4);
    tuple.put("amount", 10.0);
    Kryo kryo = new Kryo();
    oper.input.process(kryo.copy(tuple));
    oper.endWindow();
    oper.deactivate();
    /* Number of tuple, emitted */
    Assert.assertEquals("Number of tuple emitted ", 1, sink.collectedTuples.size());
    Map<String, Object> emitted = sink.collectedTuples.iterator().next();
    /* The fields present in original event is kept as it is */
    Assert.assertEquals("Number of fields in emitted tuple", 4, emitted.size());
    Assert.assertEquals("value of productId is 3", tuple.get("productId"), emitted.get("productId"));
    Assert.assertEquals("value of channelId is 4", tuple.get("channelId"), emitted.get("channelId"));
    Assert.assertEquals("value of amount is 10.0", tuple.get("amount"), emitted.get("amount"));
    /* Check if productCategory is added to the event */
    Assert.assertEquals("productCategory is part of tuple", true, emitted.containsKey("productCategory"));
    Assert.assertEquals("value of product category is 1", 5, emitted.get("productCategory"));
    Assert.assertTrue(emitted.get("productCategory") instanceof Integer);
}

Example 60 with CollectorTestSink

use of org.apache.apex.malhar.lib.testbench.CollectorTestSink in project apex-malhar by apache.

the class MapEnricherTest method includeSelectedKeys.

@Test
public void includeSelectedKeys() {
    MapEnricher oper = new MapEnricher();
    oper.setStore(new MemoryStore());
    oper.setLookupFields(Arrays.asList("In1"));
    oper.setIncludeFields(Arrays.asList("A", "B"));
    oper.setup(null);
    CollectorTestSink sink = new CollectorTestSink();
    TestUtils.setSink(oper.output, sink);
    Map<String, Object> inMap = Maps.newHashMap();
    inMap.put("In1", "Value1");
    inMap.put("In2", "Value2");
    oper.activate(null);
    oper.beginWindow(1);
    oper.input.process(inMap);
    oper.endWindow();
    oper.deactivate();
    Assert.assertEquals("includeSelectedKeys: Number of tuples emitted: ", 1, sink.collectedTuples.size());
    Assert.assertEquals("Enrich Tuple: ", "{A=Val_A, B=Val_B, In2=Value2, In1=Value1}", sink.collectedTuples.get(0).toString());
}