Examples with TimeData org.apache.storm.st.topology.window.data.TimeData used on opensource projects

Example 1 with TimeData

use of org.apache.storm.st.topology.window.data.TimeData in project storm by apache.

the class SlidingWindowTest method runAndVerifyTime.

static void runAndVerifyTime(int windowSec, int slideSec, TestableTopology testable, TopoWrap topo) throws TException, java.net.MalformedURLException {
    topo.submitSuccessfully();
    final int minSpoutEmits = 1000 + windowSec;
    final int minBoltEmits = 5;
    String boltName = testable.getBoltName();
    String spoutName = testable.getSpoutName();
    topo.waitForProgress(minSpoutEmits, spoutName, 60 + 10 * (windowSec + slideSec));
    topo.waitForProgress(minBoltEmits, boltName, 60 + 10 * (windowSec + slideSec));
    final List<TimeData> allSpoutData = topo.getLogData(spoutName, TimeData.CLS);
    final List<LogData> allBoltLog = topo.getLogData(boltName);
    final List<TimeDataWindow> allBoltData = topo.getLogData(boltName, TimeDataWindow.CLS);
    Assert.assertTrue(allBoltLog.size() >= minBoltEmits, "Expecting min " + minBoltEmits + " bolt emits, found: " + allBoltLog.size() + " \n\t" + allBoltLog);
    final DateTime firstEndTime = TimeUtil.ceil(new DateTime(allSpoutData.get(0).getDate()).withZone(DateTimeZone.UTC), slideSec);
    final int numberOfWindows = allBoltLog.size() - windowSec / slideSec;
    for (int i = 0; i < numberOfWindows; ++i) {
        final DateTime toDate = firstEndTime.plusSeconds(i * slideSec);
        final DateTime fromDate = toDate.minusSeconds(windowSec);
        log.info("Comparing window: " + fromDate + " to " + toDate + " iter " + (i + 1) + "/" + numberOfWindows);
        final TimeDataWindow computedWindow = TimeDataWindow.newInstance(allSpoutData, fromDate, toDate);
        final LogData oneBoltLog = allBoltLog.get(i);
        final TimeDataWindow actualWindow = allBoltData.get(i);
        log.info("Actual window: " + actualWindow.getDescription());
        log.info("Computed window: " + computedWindow.getDescription());
        for (TimeData oneLog : computedWindow) {
            Assert.assertTrue(actualWindow.contains(oneLog), String.format("Missing: '%s' \n\tActual: '%s' \n\tComputed window: '%s'", oneLog, oneBoltLog, computedWindow));
        }
        for (TimeData oneLog : actualWindow) {
            Assert.assertTrue(computedWindow.contains(oneLog), String.format("Extra: '%s' \n\tActual: '%s' \n\tComputed window: '%s'", oneLog, oneBoltLog, computedWindow));
        }
    }
}

Example 2 with TimeData

use of org.apache.storm.st.topology.window.data.TimeData in project storm by apache.

the class WindowVerifier method runAndVerifyTime.

/**
 * Run the topology and verify that the number and contents of time based windows is as expected
 * once the spout and bolt have emitted sufficient tuples.
 * The spout and bolt are required to log exactly one log line per emit/window using {@link StringDecorator}
 */
public void runAndVerifyTime(int windowSec, int slideSec, TestableTopology testable, TopoWrap topo) throws IOException, TException, java.net.MalformedURLException {
    topo.submitSuccessfully();
    final int minSpoutEmits = 100;
    final int minBoltEmits = 5;
    String boltName = testable.getBoltName();
    String spoutName = testable.getSpoutName();
    // Waiting for spout tuples isn't strictly necessary since we also wait for bolt emits, but do it anyway
    // Allow two minutes for topology startup, then wait for at most the time it should take to produce 10 windows
    topo.assertProgress(minSpoutEmits, testable.getSpoutExecutors(), spoutName, 180 + 10 * slideSec);
    topo.assertProgress(minBoltEmits, testable.getBoltExecutors(), boltName, 180 + 10 * slideSec);
    final List<TimeData> allSpoutLogLines = topo.getDeserializedDecoratedLogLines(spoutName, TimeData::fromJson);
    final List<TimeDataWindow> allBoltLogLines = topo.getDeserializedDecoratedLogLines(boltName, TimeDataWindow::fromJson);
    Assert.assertTrue(allBoltLogLines.size() >= minBoltEmits, "Expecting min " + minBoltEmits + " bolt emits, found: " + allBoltLogLines.size() + " \n\t" + allBoltLogLines);
    final DateTime firstWindowEndTime = TimeUtil.ceil(new DateTime(allSpoutLogLines.get(0).getDate()).withZone(DateTimeZone.UTC), slideSec);
    final int numberOfWindows = allBoltLogLines.size();
    /*
         * Windows should be aligned to the slide size, starting at firstWindowEndTime - windowSec.
         * Because all windows are aligned to the slide size, we can partition the spout emitted timestamps by which window they should fall in.
         * This checks that the partitioned spout emits fall in the expected windows, based on the logs from the spout and bolt.
         */
    for (int i = 0; i < numberOfWindows; ++i) {
        final DateTime windowEnd = firstWindowEndTime.plusSeconds(i * slideSec);
        final DateTime windowStart = windowEnd.minusSeconds(windowSec);
        LOG.info("Comparing window: " + windowStart + " to " + windowEnd + " iter " + (i + 1) + "/" + numberOfWindows);
        final List<TimeData> expectedSpoutEmitsInWindow = allSpoutLogLines.stream().filter(spoutLog -> {
            DateTime spoutLogTime = new DateTime(spoutLog.getDate());
            // The window boundaries are )windowStart, windowEnd)
            return spoutLogTime.isAfter(windowStart) && spoutLogTime.isBefore(windowEnd.plusMillis(1));
        }).collect(Collectors.toList());
        TimeDataWindow expectedWindow = new TimeDataWindow(expectedSpoutEmitsInWindow);
        final TimeDataWindow actualWindow = allBoltLogLines.get(i);
        LOG.info("Actual window: " + actualWindow.getDescription());
        LOG.info("Expected window: " + expectedWindow.getDescription());
        for (TimeData oneLog : expectedWindow.getTimeData()) {
            Assertions.assertTrue(actualWindow.getTimeData().contains(oneLog), () -> String.format("Missing: '%s' \n\tActual: '%s' \n\tComputed window: '%s'", oneLog, actualWindow, expectedWindow));
        }
        for (TimeData oneLog : actualWindow.getTimeData()) {
            Assertions.assertTrue(expectedWindow.getTimeData().contains(oneLog), () -> String.format("Extra: '%s' \n\tActual: '%s' \n\tComputed window: '%s'", oneLog, actualWindow, expectedWindow));
        }
    }
}

Example 3 with TimeData

use of org.apache.storm.st.topology.window.data.TimeData in project storm by apache.

the class TimeDataIncrementingSpout method nextTuple.

@Override
public void nextTuple() {
    if (currentNum >= TestableTopology.MAX_SPOUT_EMITS) {
        // Stop emitting at a certain point, because log rolling breaks the tests.
        return;
    }
    // Sleep a bit between emits to ensure that we don't reach the cap too quickly, since this spout is used to test time based windows
    TimeUtil.sleepMilliSec(TestableTopology.TIMEDATA_SLEEP_BETWEEN_EMITS_MS);
    currentNum++;
    TimeData data = TimeData.newData(currentNum);
    final Values tuple = data.getValues();
    collector.emit(tuple);
    LOG.info(StringDecorator.decorate(componentId, data.toString()));
}

Example 4 with TimeData

use of org.apache.storm.st.topology.window.data.TimeData in project storm by apache.

the class TimeDataVerificationBolt method execute.

@Override
public void execute(TupleWindow inputWindow) {
    List<Tuple> tuplesInWindow = inputWindow.get();
    List<Tuple> newTuples = inputWindow.getNew();
    List<Tuple> expiredTuples = inputWindow.getExpired();
    LOG.info("tuplesInWindow.size() = " + tuplesInWindow.size());
    LOG.info("newTuples.size() = " + newTuples.size());
    LOG.info("expiredTuples.size() = " + expiredTuples.size());
    List<TimeData> dataInWindow = tuplesInWindow.stream().map(TimeData::fromTuple).collect(Collectors.toList());
    final String jsonData = TimeData.toString(dataInWindow);
    LOG.info(StringDecorator.decorate(componentId, jsonData));
    collector.emit(new Values("dummyValue"));
}