Examples with NettyTextMessageStreamGenerator edu.snu.mist.common.stream.textmessage.NettyTextMessageStreamGenerator used on opensource projects

Example 1 with NettyTextMessageStreamGenerator

use of edu.snu.mist.common.stream.textmessage.NettyTextMessageStreamGenerator in project mist by snuspl.

the class NettySourceTest method testPunctuatedNettyTextSource.

/**
 * Test whether the created source using DataGenerator by NettyTextDataGeneratorFactory receive event-time data
 * correctly from netty server, and generate proper punctuated watermark and outputs.
 * It creates 4 sources each having data generator using Netty server.
 * @throws Exception
 */
@Test(timeout = 4000L)
public void testPunctuatedNettyTextSource() throws Exception {
    final int numSources = 4;
    final int numData = 3;
    final int numWatermark = 2;
    final List<String> inputStreamWithTimestamp = Arrays.asList("Lorem ipsum dolor sit amet, consectetur adipiscing elit.:100", "In in leo nec erat fringilla mattis eu non massa.:800", "Watermark:1000", "Cras quis diam suscipit, commodo enim id, pulvinar nunc.:1200", "Watermark:1500");
    final List<String> expectedDataWithoutTimestamp = Arrays.asList("Lorem ipsum dolor sit amet, consectetur adipiscing elit.", "In in leo nec erat fringilla mattis eu non massa.", "Cras quis diam suscipit, commodo enim id, pulvinar nunc.");
    final List<Long> expectedPunctuatedWatermark = Arrays.asList(1000L, 1500L);
    final CountDownLatch dataCountDownLatch = new CountDownLatch(numSources * numData);
    final CountDownLatch watermarkCountDownLatch = new CountDownLatch(numSources * numWatermark);
    final CountDownLatch channelCountDown = new CountDownLatch(numSources);
    LOG.log(Level.FINE, "Count down data: {0}", dataCountDownLatch);
    LOG.log(Level.FINE, "Count down watermark: {0}", watermarkCountDownLatch);
    // create netty server
    try (final NettyTextMessageStreamGenerator textMessageStreamGenerator = new NettyTextMessageStreamGenerator(SERVER_ADDR, SERVER_PORT, new TestChannelHandler(channelCountDown))) {
        final Injector injector = Tang.Factory.getTang().newInjector();
        // source list
        final List<Tuple<DataGenerator, EventGenerator>> sources = new LinkedList<>();
        // result data list
        final List<List<String>> punctuatedDataResults = new LinkedList<>();
        // result watermark list
        final List<List<Long>> punctuatedWatermarkResults = new LinkedList<>();
        // Create sources having punctuated watermark
        for (int i = 0; i < numSources; i++) {
            final DataGenerator<String> dataGenerator = new NettyTextDataGenerator(SERVER_ADDR, SERVER_PORT, nettySharedResource);
            final MISTFunction<String, Tuple<String, Long>> extractFunc = (input) -> new Tuple<>(input.toString().split(":")[0], Long.parseLong(input.toString().split(":")[1]));
            final MISTPredicate<String> isWatermark = (input) -> input.toString().split(":")[0].equals("Watermark");
            final WatermarkTimestampFunction<String> parseTsFunc = (input) -> Long.parseLong(input.toString().split(":")[1]);
            final EventGenerator<String> eventGenerator = new PunctuatedEventGenerator<>(extractFunc, isWatermark, parseTsFunc, 0, null, null);
            sources.add(new Tuple<>(dataGenerator, eventGenerator));
            dataGenerator.setEventGenerator(eventGenerator);
            final List<String> receivedData = new LinkedList<>();
            final List<Long> receivedWatermark = new LinkedList<>();
            punctuatedDataResults.add(receivedData);
            punctuatedWatermarkResults.add(receivedWatermark);
            eventGenerator.setOutputEmitter(new SourceTestOutputEmitter<>(receivedData, receivedWatermark, dataCountDownLatch, watermarkCountDownLatch));
        }
        // Start to receive data stream from stream generator
        for (final Tuple<DataGenerator, EventGenerator> source : sources) {
            // start event generator
            source.getValue().start();
            // start data generator
            source.getKey().start();
        }
        // Wait until all sources connect to stream generator
        channelCountDown.await();
        inputStreamWithTimestamp.forEach(textMessageStreamGenerator::write);
        // Wait until all data are sent to source
        dataCountDownLatch.await();
        watermarkCountDownLatch.await();
        for (final List<String> received : punctuatedDataResults) {
            Assert.assertEquals(expectedDataWithoutTimestamp, received);
        }
        for (final List<Long> received : punctuatedWatermarkResults) {
            Assert.assertEquals(expectedPunctuatedWatermark, received);
        }
        // Closes
        for (final Tuple<DataGenerator, EventGenerator> source : sources) {
            // stop data generator
            source.getKey().close();
            // stop event generator
            source.getValue().close();
        }
    }
}

Example 2 with NettyTextMessageStreamGenerator

use of edu.snu.mist.common.stream.textmessage.NettyTextMessageStreamGenerator in project mist by snuspl.

the class NettySourceTest method testPeriodicNettyTextSource.

/**
 * Test whether the created sources by NettyTextDataGeneratorFactory receive processing-time data
 * correctly from netty server, and generate periodic watermark.
 * It creates a netty source server and 4 receivers.
 * @throws Exception
 */
@Test(timeout = 4000L)
public void testPeriodicNettyTextSource() throws Exception {
    final int numSources = 1;
    final int numData = 3;
    final int numWatermarkToWait = 5;
    final long period = 100;
    final long epsilon = 10;
    final List<String> inputStream = Arrays.asList("Lorem ipsum dolor sit amet, consectetur adipiscing elit.", "In in leo nec erat fringilla mattis eu non massa.", "Cras quis diam suscipit, commodo enim id, pulvinar nunc.");
    final CountDownLatch dataCountDownLatch = new CountDownLatch(numSources * numData);
    final CountDownLatch watermarkCountDownLatch = new CountDownLatch(numSources * numWatermarkToWait);
    final CountDownLatch channelCountDown = new CountDownLatch(numSources);
    LOG.log(Level.FINE, "Count down data: {0}", dataCountDownLatch);
    LOG.log(Level.FINE, "Count down watermark: {0}", watermarkCountDownLatch);
    // create netty server
    try (final NettyTextMessageStreamGenerator textMessageStreamGenerator = new NettyTextMessageStreamGenerator(SERVER_ADDR, SERVER_PORT, new TestChannelHandler(channelCountDown))) {
        // Create source having periodic watermark
        final DataGenerator<String> dataGenerator = new NettyTextDataGenerator(SERVER_ADDR, SERVER_PORT, nettySharedResource);
        final EventGenerator<String> eventGenerator = new PeriodicEventGenerator<>(null, period, 0, period, TimeUnit.MILLISECONDS, scheduler);
        dataGenerator.setEventGenerator(eventGenerator);
        final List<String> periodicReceivedData = new LinkedList<>();
        final List<Long> periodicReceivedWatermark = new LinkedList<>();
        eventGenerator.setOutputEmitter(new SourceTestOutputEmitter<>(periodicReceivedData, periodicReceivedWatermark, dataCountDownLatch, watermarkCountDownLatch));
        // Start to receive data stream from stream generator
        eventGenerator.start();
        dataGenerator.start();
        // Wait until all sources connect to stream generator
        channelCountDown.await();
        inputStream.forEach(textMessageStreamGenerator::write);
        // Wait until all data are sent to source
        dataCountDownLatch.await();
        watermarkCountDownLatch.await();
        Assert.assertEquals(inputStream, periodicReceivedData);
        Long lastTimestamp = 0L;
        for (final Long timestamp : periodicReceivedWatermark) {
            if (lastTimestamp != 0L) {
                Assert.assertTrue(Math.abs(period - (timestamp - lastTimestamp)) < epsilon);
            }
            lastTimestamp = timestamp;
        }
        // Closes
        eventGenerator.close();
        dataGenerator.close();
    }
}

Example 3 with NettyTextMessageStreamGenerator

use of edu.snu.mist.common.stream.textmessage.NettyTextMessageStreamGenerator in project mist by snuspl.

the class GroupRecoveryTest method testSingleQueryRecovery.

@Test(timeout = 500000)
public void testSingleQueryRecovery() throws Exception {
    // Start source servers.
    final CountDownLatch sourceCountDownLatch1 = new CountDownLatch(1);
    final NettyTextMessageStreamGenerator textMessageStreamGenerator1 = new NettyTextMessageStreamGenerator(SERVER_ADDR, SOURCE_PORT1, new TestChannelHandler(sourceCountDownLatch1));
    final TestSinkHandler handler1 = new TestSinkHandler();
    final NettyTextMessageOutputReceiver receiver1 = new NettyTextMessageOutputReceiver("localhost", SINK_PORT, handler1);
    // Submit query.
    final MISTQuery query = buildQuery();
    // Generate avro chained dag : needed parts from MISTExamplesUtils.submitQuery()
    final Tuple<List<AvroVertex>, List<Edge>> initialAvroOpChainDag = query.getAvroOperatorDag();
    final String appId = "testApp";
    final String queryId = "testQuery";
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroDag = avroDagBuilder.setAppId(appId).setQueryId(queryId).setJarPaths(new ArrayList<>()).setAvroVertices(initialAvroOpChainDag.getKey()).setEdges(initialAvroOpChainDag.getValue()).build();
    // Build QueryManager.
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(PeriodicCheckpointPeriod.class, "1000");
    jcb.bindImplementation(QueryManager.class, GroupAwareQueryManagerImpl.class);
    jcb.bindImplementation(QueryStarter.class, ImmediateQueryMergingStarter.class);
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    injector.bindVolatileInstance(GroupIdRequestor.class, new TestGroupIdRequestor());
    injector.bindVolatileInstance(TaskStatsUpdater.class, mock(TaskStatsUpdater.class));
    final CheckpointManager checkpointManager = injector.getInstance(CheckpointManager.class);
    final QueryManager queryManager = injector.getInstance(QueryManager.class);
    queryManager.createApplication(appId, Arrays.asList(""));
    queryManager.create(avroDag);
    // Wait until all sources connect to stream generator
    sourceCountDownLatch1.await();
    final ExecutionDags executionDags = checkpointManager.getApplication(appId).getGroups().get(0).getExecutionDags();
    Assert.assertEquals(executionDags.values().size(), 1);
    final ExecutionDag executionDag = executionDags.values().iterator().next();
    // 1st stage. Push inputs to all sources and see if the results are proper.
    SRC0INPUT1.forEach(textMessageStreamGenerator1::write);
    LATCH1.await();
    Assert.assertEquals("{aa=2, bb=1, cc=1}", handler1.getResults().get(handler1.getResults().size() - 1));
    // Sleep 2 seconds for the checkpoint events to be sent out.
    sleep(2000);
    // Checkpoint the entire MISTTask, delete it, and restore it to see if it works.
    final String groupId = checkpointManager.getApplication(appId).getGroups().get(0).getGroupId();
    checkpointManager.checkpointGroup(groupId);
    checkpointManager.deleteGroup(groupId);
    // Close the generator.
    textMessageStreamGenerator1.close();
    receiver1.close();
    // Restart source servers.
    final CountDownLatch sourceCountDownLatch2 = new CountDownLatch(1);
    final NettyTextMessageStreamGenerator textMessageStreamGenerator2 = new NettyTextMessageStreamGenerator(SERVER_ADDR, SOURCE_PORT1, new TestChannelHandler(sourceCountDownLatch2));
    final TestSinkHandler handler2 = new TestSinkHandler();
    final NettyTextMessageOutputReceiver receiver2 = new NettyTextMessageOutputReceiver("localhost", SINK_PORT, handler2);
    // Recover the group.
    checkpointManager.recoverGroup(groupId);
    // Wait until all sources connect to stream generator
    sourceCountDownLatch2.await();
    final ExecutionDags executionDags2 = checkpointManager.getApplication(appId).getGroups().get(0).getExecutionDags();
    Assert.assertEquals(executionDags2.values().size(), 1);
    final ExecutionDag executionDag2 = executionDags2.values().iterator().next();
    // 2nd stage. Push inputs to the recovered the query to see if it works.
    SRC0INPUT2.forEach(textMessageStreamGenerator2::write);
    LATCH2.await();
    Assert.assertEquals("{aa=3, bb=3, cc=3}", handler2.getResults().get(handler2.getResults().size() - 1));
    // Close the generators.
    textMessageStreamGenerator2.close();
    receiver2.close();
}