Examples with LogData org.apache.storm.st.wrapper.LogData used on opensource projects

Example 1 with LogData

use of org.apache.storm.st.wrapper.LogData in project storm by apache.

the class SlidingWindowTest method runAndVerifyCount.

static void runAndVerifyCount(int windowSize, int slideSize, TestableTopology testable, TopoWrap topo) throws TException, MalformedURLException {
    topo.submitSuccessfully();
    final int minSpoutEmits = 1000 + windowSize;
    final int minBoltEmits = 5;
    String boltName = testable.getBoltName();
    String spoutName = testable.getSpoutName();
    topo.waitForProgress(minSpoutEmits, spoutName, 180);
    topo.waitForProgress(minBoltEmits, boltName, 180);
    List<TopoWrap.ExecutorURL> boltUrls = topo.getLogUrls(boltName);
    log.info(boltUrls.toString());
    final List<LogData> allBoltData = topo.getLogData(boltName);
    final List<LogData> allSpoutData = topo.getLogData(spoutName);
    Assert.assertTrue(allBoltData.size() >= minBoltEmits, "Expecting min " + minBoltEmits + " bolt emits, found: " + allBoltData.size() + " \n\t" + allBoltData);
    final int numberOfWindows = allBoltData.size() - windowSize / slideSize;
    for (int i = 0; i < numberOfWindows; ++i) {
        log.info("Comparing window: " + (i + 1) + " of " + numberOfWindows);
        final int toIndex = (i + 1) * slideSize;
        final int fromIndex = toIndex - windowSize;
        final int positiveFromIndex = fromIndex > 0 ? fromIndex : 0;
        final List<LogData> windowData = allSpoutData.subList(positiveFromIndex, toIndex);
        final String actualString = allBoltData.get(i).toString();
        for (LogData oneLog : windowData) {
            final String logStr = oneLog.getData();
            Assert.assertTrue(actualString.contains(logStr), String.format("Missing: '%s' \nActual: '%s' \nCalculated window: '%s'", logStr, actualString, windowData));
        }
    }
}

Example 2 with LogData

use of org.apache.storm.st.wrapper.LogData in project storm by apache.

the class SlidingWindowTest method runAndVerifyTime.

static void runAndVerifyTime(int windowSec, int slideSec, TestableTopology testable, TopoWrap topo) throws TException, java.net.MalformedURLException {
    topo.submitSuccessfully();
    final int minSpoutEmits = 1000 + windowSec;
    final int minBoltEmits = 5;
    String boltName = testable.getBoltName();
    String spoutName = testable.getSpoutName();
    topo.waitForProgress(minSpoutEmits, spoutName, 60 + 10 * (windowSec + slideSec));
    topo.waitForProgress(minBoltEmits, boltName, 60 + 10 * (windowSec + slideSec));
    final List<TimeData> allSpoutData = topo.getLogData(spoutName, TimeData.CLS);
    final List<LogData> allBoltLog = topo.getLogData(boltName);
    final List<TimeDataWindow> allBoltData = topo.getLogData(boltName, TimeDataWindow.CLS);
    Assert.assertTrue(allBoltLog.size() >= minBoltEmits, "Expecting min " + minBoltEmits + " bolt emits, found: " + allBoltLog.size() + " \n\t" + allBoltLog);
    final DateTime firstEndTime = TimeUtil.ceil(new DateTime(allSpoutData.get(0).getDate()).withZone(DateTimeZone.UTC), slideSec);
    final int numberOfWindows = allBoltLog.size() - windowSec / slideSec;
    for (int i = 0; i < numberOfWindows; ++i) {
        final DateTime toDate = firstEndTime.plusSeconds(i * slideSec);
        final DateTime fromDate = toDate.minusSeconds(windowSec);
        log.info("Comparing window: " + fromDate + " to " + toDate + " iter " + (i + 1) + "/" + numberOfWindows);
        final TimeDataWindow computedWindow = TimeDataWindow.newInstance(allSpoutData, fromDate, toDate);
        final LogData oneBoltLog = allBoltLog.get(i);
        final TimeDataWindow actualWindow = allBoltData.get(i);
        log.info("Actual window: " + actualWindow.getDescription());
        log.info("Computed window: " + computedWindow.getDescription());
        for (TimeData oneLog : computedWindow) {
            Assert.assertTrue(actualWindow.contains(oneLog), String.format("Missing: '%s' \n\tActual: '%s' \n\tComputed window: '%s'", oneLog, oneBoltLog, computedWindow));
        }
        for (TimeData oneLog : actualWindow) {
            Assert.assertTrue(computedWindow.contains(oneLog), String.format("Extra: '%s' \n\tActual: '%s' \n\tComputed window: '%s'", oneLog, oneBoltLog, computedWindow));
        }
    }
}