Examples with WatermarkTimestampFunction edu.snu.mist.common.functions.WatermarkTimestampFunction used on opensource projects

Example 1 with WatermarkTimestampFunction

use of edu.snu.mist.common.functions.WatermarkTimestampFunction in project mist by snuspl.

the class PhysicalObjectGenerator method newEventGenerator.

/**
 * Get a new event generator.
 * @param conf configuration
 * @param classLoader external class loader
 * @param <T> event type
 * @return event generator
 */
@SuppressWarnings("unchecked")
public <T> EventGenerator<T> newEventGenerator(final Map<String, String> conf, final ClassLoader classLoader) throws IOException, ClassNotFoundException {
    final String type = conf.get(ConfKeys.Watermark.EVENT_GENERATOR.name());
    final String tefString = conf.get(ConfKeys.SourceConf.TIMESTAMP_EXTRACT_FUNC.name());
    final MISTFunction timestampExtractFunc;
    if (tefString == null) {
        timestampExtractFunc = null;
    } else {
        timestampExtractFunc = SerializeUtils.deserializeFromString(conf.get(ConfKeys.SourceConf.TIMESTAMP_EXTRACT_FUNC.name()), classLoader);
    }
    if (type.equals(ConfValues.EventGeneratorType.PERIODIC_EVENT_GEN.name())) {
        // periodic event generator
        final long period = Long.valueOf(conf.get(ConfKeys.Watermark.PERIODIC_WATERMARK_PERIOD.name()));
        final long delay = Long.valueOf(conf.get(ConfKeys.Watermark.PERIODIC_WATERMARK_DELAY.name()));
        return new PeriodicEventGenerator(timestampExtractFunc, period, checkpointPeriod, delay, watermarkTimeUnit, scheduler);
    } else if (type.equals(ConfValues.EventGeneratorType.PUNCTUATED_EVENT_GEN.name())) {
        // punctuated event generator
        final MISTPredicate watermarkPredicate = SerializeUtils.deserializeFromString(conf.get(ConfKeys.Watermark.WATERMARK_PREDICATE.name()), classLoader);
        final WatermarkTimestampFunction tf = SerializeUtils.deserializeFromString(conf.get(ConfKeys.Watermark.TIMESTAMP_PARSE_OBJECT.name()), classLoader);
        return new PunctuatedEventGenerator(timestampExtractFunc, watermarkPredicate, tf, checkpointPeriod, watermarkTimeUnit, scheduler);
    } else {
        throw new RuntimeException("Invalid event generator: " + type);
    }
}

Example 2 with WatermarkTimestampFunction

use of edu.snu.mist.common.functions.WatermarkTimestampFunction in project mist by snuspl.

the class NettySourceTest method testPunctuatedNettyTextSource.

/**
 * Test whether the created source using DataGenerator by NettyTextDataGeneratorFactory receive event-time data
 * correctly from netty server, and generate proper punctuated watermark and outputs.
 * It creates 4 sources each having data generator using Netty server.
 * @throws Exception
 */
@Test(timeout = 4000L)
public void testPunctuatedNettyTextSource() throws Exception {
    final int numSources = 4;
    final int numData = 3;
    final int numWatermark = 2;
    final List<String> inputStreamWithTimestamp = Arrays.asList("Lorem ipsum dolor sit amet, consectetur adipiscing elit.:100", "In in leo nec erat fringilla mattis eu non massa.:800", "Watermark:1000", "Cras quis diam suscipit, commodo enim id, pulvinar nunc.:1200", "Watermark:1500");
    final List<String> expectedDataWithoutTimestamp = Arrays.asList("Lorem ipsum dolor sit amet, consectetur adipiscing elit.", "In in leo nec erat fringilla mattis eu non massa.", "Cras quis diam suscipit, commodo enim id, pulvinar nunc.");
    final List<Long> expectedPunctuatedWatermark = Arrays.asList(1000L, 1500L);
    final CountDownLatch dataCountDownLatch = new CountDownLatch(numSources * numData);
    final CountDownLatch watermarkCountDownLatch = new CountDownLatch(numSources * numWatermark);
    final CountDownLatch channelCountDown = new CountDownLatch(numSources);
    LOG.log(Level.FINE, "Count down data: {0}", dataCountDownLatch);
    LOG.log(Level.FINE, "Count down watermark: {0}", watermarkCountDownLatch);
    // create netty server
    try (final NettyTextMessageStreamGenerator textMessageStreamGenerator = new NettyTextMessageStreamGenerator(SERVER_ADDR, SERVER_PORT, new TestChannelHandler(channelCountDown))) {
        final Injector injector = Tang.Factory.getTang().newInjector();
        // source list
        final List<Tuple<DataGenerator, EventGenerator>> sources = new LinkedList<>();
        // result data list
        final List<List<String>> punctuatedDataResults = new LinkedList<>();
        // result watermark list
        final List<List<Long>> punctuatedWatermarkResults = new LinkedList<>();
        // Create sources having punctuated watermark
        for (int i = 0; i < numSources; i++) {
            final DataGenerator<String> dataGenerator = new NettyTextDataGenerator(SERVER_ADDR, SERVER_PORT, nettySharedResource);
            final MISTFunction<String, Tuple<String, Long>> extractFunc = (input) -> new Tuple<>(input.toString().split(":")[0], Long.parseLong(input.toString().split(":")[1]));
            final MISTPredicate<String> isWatermark = (input) -> input.toString().split(":")[0].equals("Watermark");
            final WatermarkTimestampFunction<String> parseTsFunc = (input) -> Long.parseLong(input.toString().split(":")[1]);
            final EventGenerator<String> eventGenerator = new PunctuatedEventGenerator<>(extractFunc, isWatermark, parseTsFunc, 0, null, null);
            sources.add(new Tuple<>(dataGenerator, eventGenerator));
            dataGenerator.setEventGenerator(eventGenerator);
            final List<String> receivedData = new LinkedList<>();
            final List<Long> receivedWatermark = new LinkedList<>();
            punctuatedDataResults.add(receivedData);
            punctuatedWatermarkResults.add(receivedWatermark);
            eventGenerator.setOutputEmitter(new SourceTestOutputEmitter<>(receivedData, receivedWatermark, dataCountDownLatch, watermarkCountDownLatch));
        }
        // Start to receive data stream from stream generator
        for (final Tuple<DataGenerator, EventGenerator> source : sources) {
            // start event generator
            source.getValue().start();
            // start data generator
            source.getKey().start();
        }
        // Wait until all sources connect to stream generator
        channelCountDown.await();
        inputStreamWithTimestamp.forEach(textMessageStreamGenerator::write);
        // Wait until all data are sent to source
        dataCountDownLatch.await();
        watermarkCountDownLatch.await();
        for (final List<String> received : punctuatedDataResults) {
            Assert.assertEquals(expectedDataWithoutTimestamp, received);
        }
        for (final List<Long> received : punctuatedWatermarkResults) {
            Assert.assertEquals(expectedPunctuatedWatermark, received);
        }
        // Closes
        for (final Tuple<DataGenerator, EventGenerator> source : sources) {
            // stop data generator
            source.getKey().close();
            // stop event generator
            source.getValue().close();
        }
    }
}