Examples with LogstreamPropertyRegistry com.datatorrent.apps.logstream.PropertyRegistry.LogstreamPropertyRegistry used on opensource projects

Example 1 with LogstreamPropertyRegistry

use of com.datatorrent.apps.logstream.PropertyRegistry.LogstreamPropertyRegistry in project apex-malhar by apache.

the class Application1 method populateDAG.

@Override
public void populateDAG(DAG dag, Configuration c) {
    int topNtupleCount = 10;
    LogstreamPropertyRegistry registry = new LogstreamPropertyRegistry();
    // set app name
    dag.setAttribute(DAG.APPLICATION_NAME, "Logstream Application");
    dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, 500);
    RabbitMQLogsInputOperator logInput = dag.addOperator("LogInput", new RabbitMQLogsInputOperator());
    logInput.setRegistry(registry);
    logInput.addPropertiesFromString(new String[] { "localhost:5672", "logsExchange", "direct", "apache:mysql:syslog:system" });
    JsonByteArrayOperator jsonToMap = dag.addOperator("JsonToMap", new JsonByteArrayOperator());
    jsonToMap.setConcatenationCharacter('_');
    FilterOperator filterOperator = dag.addOperator("FilterOperator", new FilterOperator());
    filterOperator.setRegistry(registry);
    filterOperator.addFilterCondition(new String[] { "type=apache", "response", "response.equals(\"404\")" });
    filterOperator.addFilterCondition(new String[] { "type=apache", "agentinfo_name", "agentinfo_name.equals(\"Firefox\")" });
    filterOperator.addFilterCondition(new String[] { "type=apache", "default=true" });
    filterOperator.addFilterCondition(new String[] { "type=mysql", "default=true" });
    filterOperator.addFilterCondition(new String[] { "type=syslog", "default=true" });
    filterOperator.addFilterCondition(new String[] { "type=system", "default=true" });
    DimensionOperator dimensionOperator = dag.addOperator("DimensionOperator", new DimensionOperator());
    dimensionOperator.setRegistry(registry);
    String[] dimensionInputString1 = new String[] { "type=apache", "timebucket=s", "dimensions=request", "dimensions=clientip", "dimensions=clientip:request", "values=bytes.sum:bytes.avg" };
    // String[] dimensionInputString1 = new String[] {"type=apache", "timebucket=s", "dimensions=request", "dimensions=clientip","values=bytes.sum"};
    String[] dimensionInputString2 = new String[] { "type=system", "timebucket=s", "dimensions=disk", "values=writes.avg" };
    String[] dimensionInputString3 = new String[] { "type=syslog", "timebucket=s", "dimensions=program", "values=pid.count" };
    dimensionOperator.addPropertiesFromString(dimensionInputString1);
    dimensionOperator.addPropertiesFromString(dimensionInputString2);
    dimensionOperator.addPropertiesFromString(dimensionInputString3);
    LogstreamTopN topN = dag.addOperator("TopN", new LogstreamTopN());
    topN.setN(topNtupleCount);
    topN.setRegistry(registry);
    LogstreamWidgetOutputOperator widgetOut = dag.addOperator("WidgetOut", new LogstreamWidgetOutputOperator());
    widgetOut.logstreamTopNInput.setN(topNtupleCount);
    widgetOut.setRegistry(registry);
    ConsoleOutputOperator consoleOut = dag.addOperator("ConsoleOut", new ConsoleOutputOperator());
    dag.addStream("inputJSonToMap", logInput.outputPort, jsonToMap.input);
    dag.addStream("toFilterOper", jsonToMap.outputFlatMap, filterOperator.input);
    dag.addStream("toDimensionOper", filterOperator.outputMap, dimensionOperator.in);
    dag.addStream("toTopN", dimensionOperator.aggregationsOutput, topN.data);
    dag.addStream("toWS", topN.top, widgetOut.logstreamTopNInput, consoleOut.input);
    dag.setInputPortAttribute(jsonToMap.input, PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(filterOperator.input, PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(consoleOut.input, PortContext.PARTITION_PARALLEL, true);
}

Example 2 with LogstreamPropertyRegistry

use of com.datatorrent.apps.logstream.PropertyRegistry.LogstreamPropertyRegistry in project apex-malhar by apache.

the class FilterOperatorTest method testOperator.

@Test
@SuppressWarnings("unchecked")
public void testOperator() {
    FilterOperator oper = new FilterOperator();
    LogstreamPropertyRegistry registry = new LogstreamPropertyRegistry();
    registry.bind(LogstreamUtil.LOG_TYPE, "apache");
    oper.setRegistry(registry);
    oper.setup(null);
    String filter1 = "a==\"1\"&&b==\"2\"&&c_info==\"abc\"";
    oper.addFilterCondition(new String[] { "type=apache", "a", "b", "c_info", "d", filter1 });
    String filter2 = "d==1";
    oper.addFilterCondition(new String[] { "type=apache", "d", filter2 });
    String filter3 = "a==\"1\"";
    oper.addFilterCondition(new String[] { "type=apache", "a", filter3 });
    String filter4 = "e==\"2\"";
    oper.addFilterCondition(new String[] { "type=apache", "e", filter4 });
    String filter5 = "response.equals(\"404\")";
    oper.addFilterCondition(new String[] { "type=apache", "response", filter5 });
    String filter6 = "default=true";
    oper.addFilterCondition(new String[] { "type=apache", filter6 });
    HashMap<String, Object> inMap = new HashMap<String, Object>();
    inMap.put(LogstreamUtil.LOG_TYPE, registry.getIndex(LogstreamUtil.LOG_TYPE, "apache"));
    inMap.put("a", "1");
    inMap.put("b", "2");
    inMap.put("c_info", "abc");
    inMap.put("d", 1);
    inMap.put("e", "3");
    inMap.put("response", "404");
    Set<Integer> expectedPassSet = new HashSet<Integer>();
    expectedPassSet.add(registry.getIndex(LogstreamUtil.FILTER, filter1));
    expectedPassSet.add(registry.getIndex(LogstreamUtil.FILTER, filter2));
    expectedPassSet.add(registry.getIndex(LogstreamUtil.FILTER, filter3));
    expectedPassSet.add(registry.getIndex(LogstreamUtil.FILTER, filter5));
    expectedPassSet.add(registry.getIndex(LogstreamUtil.FILTER, "apache_DEFAULT"));
    CollectorTestSink mapSink = new CollectorTestSink();
    oper.outputMap.setSink(mapSink);
    oper.beginWindow(0);
    oper.input.process(inMap);
    oper.endWindow();
    Assert.assertEquals("tuple count", 5, mapSink.collectedTuples.size());
    List<HashMap<String, Object>> tuples = mapSink.collectedTuples;
    Set<Integer> actualPassSet = new HashSet<Integer>();
    for (HashMap<String, Object> tuple : tuples) {
        Integer filter = (Integer) tuple.get(LogstreamUtil.FILTER);
        actualPassSet.add(filter);
    }
    Assert.assertEquals("Passed filters", expectedPassSet, actualPassSet);
}

Example 3 with LogstreamPropertyRegistry

use of com.datatorrent.apps.logstream.PropertyRegistry.LogstreamPropertyRegistry in project apex-malhar by apache.

the class LogstreamTopNTest method testOperator.

@Test
@SuppressWarnings("unchecked")
public void testOperator() {
    LogstreamTopN oper = new LogstreamTopN();
    LogstreamPropertyRegistry registry = new LogstreamPropertyRegistry();
    registry.bind(LogstreamUtil.LOG_TYPE, "apache");
    registry.bind(LogstreamUtil.FILTER, "default");
    oper.setRegistry(registry);
    oper.setN(5);
    CollectorTestSink mapSink = new CollectorTestSink();
    oper.top.setSink(mapSink);
    oper.beginWindow(0);
    Map<String, DimensionObject<String>> tuple1 = new HashMap<String, DimensionObject<String>>();
    DimensionObject<String> dimObja = new DimensionObject<String>(new MutableDouble(10), "a");
    tuple1.put("m|201402121900|0|65535|131075|val.COUNT", dimObja);
    oper.data.process(tuple1);
    DimensionObject<String> dimObjb = new DimensionObject<String>(new MutableDouble(1), "b");
    tuple1.put("m|201402121900|0|65535|131075|val.COUNT", dimObjb);
    oper.data.process(tuple1);
    DimensionObject<String> dimObjc = new DimensionObject<String>(new MutableDouble(5), "c");
    tuple1.put("m|201402121900|0|65535|131075|val.COUNT", dimObjc);
    oper.data.process(tuple1);
    DimensionObject<String> dimObjd = new DimensionObject<String>(new MutableDouble(2), "d");
    tuple1.put("m|201402121900|0|65535|131075|val.COUNT", dimObjd);
    oper.data.process(tuple1);
    DimensionObject<String> dimObje = new DimensionObject<String>(new MutableDouble(15), "e");
    tuple1.put("m|201402121900|0|65535|131075|val.COUNT", dimObje);
    oper.data.process(tuple1);
    DimensionObject<String> dimObjf = new DimensionObject<String>(new MutableDouble(4), "f");
    tuple1.put("m|201402121900|0|65535|131075|val.COUNT", dimObjf);
    oper.data.process(tuple1);
    oper.endWindow();
    @SuppressWarnings("unchecked") Map<String, List<DimensionObject<String>>> tuples = (Map<String, List<DimensionObject<String>>>) mapSink.collectedTuples.get(0);
    List<DimensionObject<String>> outList = tuples.get("m|201402121900|0|65535|131075|val.COUNT");
    List<DimensionObject<String>> expectedList = new ArrayList<DimensionObject<String>>();
    expectedList.add(dimObje);
    expectedList.add(dimObja);
    expectedList.add(dimObjc);
    expectedList.add(dimObjf);
    expectedList.add(dimObjd);
    Assert.assertEquals("Size", expectedList.size(), outList.size());
    Assert.assertEquals("compare list", expectedList, outList);
}

Example 4 with LogstreamPropertyRegistry

use of com.datatorrent.apps.logstream.PropertyRegistry.LogstreamPropertyRegistry in project apex-malhar by apache.

the class DimensionOperatorTest method testOperator.

@Test
@SuppressWarnings("unchecked")
public void testOperator() {
    DimensionOperator oper = new DimensionOperator();
    LogstreamPropertyRegistry registry = new LogstreamPropertyRegistry();
    registry.bind(LogstreamUtil.LOG_TYPE, "apache");
    registry.bind(LogstreamUtil.FILTER, "ALL");
    oper.setRegistry(registry);
    // user input example::
    // type=apache,timebucket=m,timebucket=h,dimensions=a:b:c,dimensions=b:c,dimensions=b,dimensions=d,values=x.sum:y.sum:y.avg
    oper.addPropertiesFromString(new String[] { "type=apache", "timebucket=s", "dimensions=name", "dimensions=url", "dimensions=name:url", "values=value.sum:value.avg" });
    HashMap<String, Object> inMap1 = new HashMap<String, Object>();
    inMap1.put(LogstreamUtil.LOG_TYPE, registry.getIndex(LogstreamUtil.LOG_TYPE, "apache"));
    inMap1.put(LogstreamUtil.FILTER, registry.getIndex(LogstreamUtil.FILTER, "ALL"));
    inMap1.put("name", "abc");
    inMap1.put("url", "http://www.t.co");
    inMap1.put("value", 25);
    inMap1.put("response", "404");
    HashMap<String, Object> inMap2 = new HashMap<String, Object>();
    inMap2.put(LogstreamUtil.LOG_TYPE, registry.getIndex(LogstreamUtil.LOG_TYPE, "apache"));
    inMap2.put(LogstreamUtil.FILTER, registry.getIndex(LogstreamUtil.FILTER, "ALL"));
    inMap2.put("name", "xyz");
    inMap2.put("url", "http://www.t.co");
    inMap2.put("value", 25);
    inMap2.put("response", "404");
    HashMap<String, Object> inMap3 = new HashMap<String, Object>();
    inMap3.put(LogstreamUtil.LOG_TYPE, registry.getIndex(LogstreamUtil.LOG_TYPE, "apache"));
    inMap3.put(LogstreamUtil.FILTER, registry.getIndex(LogstreamUtil.FILTER, "ALL"));
    inMap3.put("name", "abc");
    inMap3.put("url", "http://www.t.co");
    inMap3.put("value", 25);
    inMap3.put("response", "404");
    HashMap<String, Object> inMap4 = new HashMap<String, Object>();
    inMap4.put(LogstreamUtil.LOG_TYPE, registry.getIndex(LogstreamUtil.LOG_TYPE, "apache"));
    inMap4.put(LogstreamUtil.FILTER, registry.getIndex(LogstreamUtil.FILTER, "ALL"));
    inMap4.put("name", "abc");
    inMap4.put("url", "http://www.t.co");
    inMap4.put("value", 25);
    inMap4.put("response", "404");
    CollectorTestSink mapSink = new CollectorTestSink();
    oper.aggregationsOutput.setSink(mapSink);
    long now = System.currentTimeMillis();
    long currentId = 0L;
    long windowId = (now / 1000) << 32 | currentId;
    oper.beginWindow(windowId);
    oper.in.process(inMap1);
    oper.in.process(inMap2);
    oper.in.process(inMap3);
    oper.in.process(inMap4);
    try {
        Thread.sleep(1000);
    } catch (Throwable ex) {
        DTThrowable.rethrow(ex);
    }
    oper.endWindow();
    currentId++;
    now = System.currentTimeMillis();
    windowId = (now / 1000) << 32 | currentId;
    oper.beginWindow(windowId);
    oper.endWindow();
    @SuppressWarnings("unchecked") List<Map<String, DimensionObject<String>>> tuples = mapSink.collectedTuples;
    for (Map<String, DimensionObject<String>> map : tuples) {
        for (Entry<String, DimensionObject<String>> entry : map.entrySet()) {
            String key = entry.getKey();
            DimensionObject<String> dimObj = entry.getValue();
            if (key.contains("COUNT")) {
                if (dimObj.getVal().equals("xyz")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(1), dimObj.getCount());
                } else if (dimObj.getVal().equals("abc")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(3), dimObj.getCount());
                } else if (dimObj.getVal().equals("abc,http://www.t.co")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(3), dimObj.getCount());
                } else if (dimObj.getVal().equals("xyz,http://www.t.co")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(1), dimObj.getCount());
                } else if (dimObj.getVal().equals("http://www.t.co")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(4), dimObj.getCount());
                } else {
                    Assert.fail("Unexpected dimension object received: " + dimObj + " for key: " + key);
                }
            } else if (key.contains("AVERAGE")) {
                if (dimObj.getVal().equals("xyz")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(25), dimObj.getCount());
                } else if (dimObj.getVal().equals("abc")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(25), dimObj.getCount());
                } else if (dimObj.getVal().equals("abc,http://www.t.co")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(25), dimObj.getCount());
                } else if (dimObj.getVal().equals("xyz,http://www.t.co")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(25), dimObj.getCount());
                } else if (dimObj.getVal().equals("http://www.t.co")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(25), dimObj.getCount());
                } else {
                    Assert.fail("Unexpected dimension object received: " + dimObj + " for key: " + key);
                }
            } else if (key.contains("SUM")) {
                if (dimObj.getVal().equals("xyz")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(25), dimObj.getCount());
                } else if (dimObj.getVal().equals("abc")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(75), dimObj.getCount());
                } else if (dimObj.getVal().equals("abc,http://www.t.co")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(75), dimObj.getCount());
                } else if (dimObj.getVal().equals("xyz,http://www.t.co")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(25), dimObj.getCount());
                } else if (dimObj.getVal().equals("http://www.t.co")) {
                    Assert.assertEquals("Count for key " + key, new MutableDouble(100), dimObj.getCount());
                } else {
                    Assert.fail("Unexpected dimension object received: " + dimObj + " for key: " + key);
                }
            } else {
                Assert.fail("Unexpected key received: " + key);
            }
        }
    }
}