Examples with Tuple org.apache.reef.io.Tuple used on opensource projects

Example 6 with Tuple

use of org.apache.reef.io.Tuple in project mist by snuspl.

the class GroupAwareQueryManagerImpl method createApplication.

@Override
public ApplicationInfo createApplication(final String appId, final List<String> paths) throws InjectionException {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(ApplicationIdentifier.class, appId);
    // TODO: Submit a single jar instead of list of jars
    jcb.bindNamedParameter(JarFilePath.class, paths.get(0));
    final String groupId = groupIdRequestor.requestGroupId(appId);
    if (groupId == null) {
        throw new RuntimeException("An error occured while getting a groupId!");
    }
    jcb.bindNamedParameter(GroupId.class, groupId);
    jcb.bindNamedParameter(PeriodicCheckpointPeriod.class, String.valueOf(checkpointPeriod));
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    injector.bindVolatileInstance(MQTTResource.class, mqttSharedResource);
    injector.bindVolatileInstance(KafkaSharedResource.class, kafkaSharedResource);
    injector.bindVolatileInstance(NettySharedResource.class, nettySharedResource);
    injector.bindVolatileInstance(QueryInfoStore.class, planStore);
    final ApplicationInfo applicationInfo = injector.getInstance(ApplicationInfo.class);
    if (applicationMap.putIfAbsent(appId, applicationInfo) == null) {
        final Group group = injector.getInstance(Group.class);
        groupAllocationTableModifier.addEvent(new WritingEvent(WritingEvent.EventType.GROUP_ADD, new Tuple<>(applicationInfo, group)));
    }
    return applicationMap.get(appId);
}

Example 7 with Tuple

use of org.apache.reef.io.Tuple in project mist by snuspl.

the class NettySourceTest method testPunctuatedNettyTextSource.

/**
 * Test whether the created source using DataGenerator by NettyTextDataGeneratorFactory receive event-time data
 * correctly from netty server, and generate proper punctuated watermark and outputs.
 * It creates 4 sources each having data generator using Netty server.
 * @throws Exception
 */
@Test(timeout = 4000L)
public void testPunctuatedNettyTextSource() throws Exception {
    final int numSources = 4;
    final int numData = 3;
    final int numWatermark = 2;
    final List<String> inputStreamWithTimestamp = Arrays.asList("Lorem ipsum dolor sit amet, consectetur adipiscing elit.:100", "In in leo nec erat fringilla mattis eu non massa.:800", "Watermark:1000", "Cras quis diam suscipit, commodo enim id, pulvinar nunc.:1200", "Watermark:1500");
    final List<String> expectedDataWithoutTimestamp = Arrays.asList("Lorem ipsum dolor sit amet, consectetur adipiscing elit.", "In in leo nec erat fringilla mattis eu non massa.", "Cras quis diam suscipit, commodo enim id, pulvinar nunc.");
    final List<Long> expectedPunctuatedWatermark = Arrays.asList(1000L, 1500L);
    final CountDownLatch dataCountDownLatch = new CountDownLatch(numSources * numData);
    final CountDownLatch watermarkCountDownLatch = new CountDownLatch(numSources * numWatermark);
    final CountDownLatch channelCountDown = new CountDownLatch(numSources);
    LOG.log(Level.FINE, "Count down data: {0}", dataCountDownLatch);
    LOG.log(Level.FINE, "Count down watermark: {0}", watermarkCountDownLatch);
    // create netty server
    try (final NettyTextMessageStreamGenerator textMessageStreamGenerator = new NettyTextMessageStreamGenerator(SERVER_ADDR, SERVER_PORT, new TestChannelHandler(channelCountDown))) {
        final Injector injector = Tang.Factory.getTang().newInjector();
        // source list
        final List<Tuple<DataGenerator, EventGenerator>> sources = new LinkedList<>();
        // result data list
        final List<List<String>> punctuatedDataResults = new LinkedList<>();
        // result watermark list
        final List<List<Long>> punctuatedWatermarkResults = new LinkedList<>();
        // Create sources having punctuated watermark
        for (int i = 0; i < numSources; i++) {
            final DataGenerator<String> dataGenerator = new NettyTextDataGenerator(SERVER_ADDR, SERVER_PORT, nettySharedResource);
            final MISTFunction<String, Tuple<String, Long>> extractFunc = (input) -> new Tuple<>(input.toString().split(":")[0], Long.parseLong(input.toString().split(":")[1]));
            final MISTPredicate<String> isWatermark = (input) -> input.toString().split(":")[0].equals("Watermark");
            final WatermarkTimestampFunction<String> parseTsFunc = (input) -> Long.parseLong(input.toString().split(":")[1]);
            final EventGenerator<String> eventGenerator = new PunctuatedEventGenerator<>(extractFunc, isWatermark, parseTsFunc, 0, null, null);
            sources.add(new Tuple<>(dataGenerator, eventGenerator));
            dataGenerator.setEventGenerator(eventGenerator);
            final List<String> receivedData = new LinkedList<>();
            final List<Long> receivedWatermark = new LinkedList<>();
            punctuatedDataResults.add(receivedData);
            punctuatedWatermarkResults.add(receivedWatermark);
            eventGenerator.setOutputEmitter(new SourceTestOutputEmitter<>(receivedData, receivedWatermark, dataCountDownLatch, watermarkCountDownLatch));
        }
        // Start to receive data stream from stream generator
        for (final Tuple<DataGenerator, EventGenerator> source : sources) {
            // start event generator
            source.getValue().start();
            // start data generator
            source.getKey().start();
        }
        // Wait until all sources connect to stream generator
        channelCountDown.await();
        inputStreamWithTimestamp.forEach(textMessageStreamGenerator::write);
        // Wait until all data are sent to source
        dataCountDownLatch.await();
        watermarkCountDownLatch.await();
        for (final List<String> received : punctuatedDataResults) {
            Assert.assertEquals(expectedDataWithoutTimestamp, received);
        }
        for (final List<Long> received : punctuatedWatermarkResults) {
            Assert.assertEquals(expectedPunctuatedWatermark, received);
        }
        // Closes
        for (final Tuple<DataGenerator, EventGenerator> source : sources) {
            // stop data generator
            source.getKey().close();
            // stop event generator
            source.getValue().close();
        }
    }
}

Example 8 with Tuple

use of org.apache.reef.io.Tuple in project mist by snuspl.

the class ImmediateQueryMergingStarterTest method generateSimpleDag.

/**
 * Generate a simple query that has the following structure: src -> operator -> sink.
 * @param source source
 * @param physicalOperator operator
 * @param sink sink
 * @return dag
 */
private Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> generateSimpleDag(final TestSource source, final PhysicalOperator physicalOperator, final PhysicalSink<String> sink, final ConfigVertex srcVertex, final ConfigVertex ocVertex, final ConfigVertex sinkVertex) throws IOException, ClassNotFoundException {
    // Create DAG
    final DAG<ConfigVertex, MISTEdge> dag = new AdjacentListConcurrentMapDAG<>();
    dag.addVertex(srcVertex);
    dag.addVertex(ocVertex);
    dag.addVertex(sinkVertex);
    dag.addEdge(srcVertex, ocVertex, new MISTEdge(Direction.LEFT));
    dag.addEdge(ocVertex, sinkVertex, new MISTEdge(Direction.LEFT));
    final DAG<ExecutionVertex, MISTEdge> exDag = new AdjacentListConcurrentMapDAG<>();
    exDag.addVertex(source);
    exDag.addVertex(physicalOperator);
    exDag.addVertex(sink);
    exDag.addEdge(source, physicalOperator, new MISTEdge(Direction.LEFT));
    exDag.addEdge(physicalOperator, sink, new MISTEdge(Direction.LEFT));
    when(executionVertexGenerator.generate(eq(srcVertex), any(URL[].class), any(ClassLoader.class))).thenReturn(source);
    when(executionVertexGenerator.generate(eq(ocVertex), any(URL[].class), any(ClassLoader.class))).thenReturn(physicalOperator);
    when(executionVertexGenerator.generate(eq(sinkVertex), any(URL[].class), any(ClassLoader.class))).thenReturn(sink);
    final ExecutionDag executionDag = new ExecutionDag(exDag);
    return new Tuple<>(dag, executionDag);
}

Example 9 with Tuple

use of org.apache.reef.io.Tuple in project mist by snuspl.

the class ConditionalBranchOperatorTest method testConditionalBranchOperator.

/**
 * Test conditional branch operation.
 * It classifies the input string according to it's length.
 */
@Test
public void testConditionalBranchOperator() throws InjectionException {
    // input stream events
    final MistDataEvent d1 = new MistDataEvent("1", 1L);
    final MistDataEvent d2 = new MistDataEvent("22", 2L);
    final MistDataEvent d3 = new MistDataEvent("333", 3L);
    final MistDataEvent d4 = new MistDataEvent("4444", 4L);
    final MistDataEvent d5 = new MistDataEvent("55555", 5L);
    final MistWatermarkEvent w1 = new MistWatermarkEvent(6L);
    // classify the string according to it's length
    final List<MISTPredicate<String>> predicates = new ArrayList<>();
    // "1" will be passed with index 1
    predicates.add((input) -> input.length() < 2);
    // "22" will be passed with index 2
    predicates.add((input) -> input.length() < 3);
    // "333", "4444" will be passed with index 3
    predicates.add((input) -> input.length() < 5);
    // "55555" will not be passed
    final List<Tuple<MistEvent, Integer>> result = new LinkedList<>();
    final ConditionalBranchOperator<String> conditionalBranchOperator = new ConditionalBranchOperator<>(predicates);
    conditionalBranchOperator.setOutputEmitter(new IndexOutputEmitter(result));
    conditionalBranchOperator.processLeftData(d1);
    conditionalBranchOperator.processLeftData(d2);
    conditionalBranchOperator.processLeftData(d3);
    conditionalBranchOperator.processLeftData(d4);
    conditionalBranchOperator.processLeftData(d5);
    conditionalBranchOperator.processLeftWatermark(w1);
    Assert.assertEquals(5, result.size());
    Assert.assertEquals(new Tuple<>(d1, 1), result.get(0));
    Assert.assertEquals(new Tuple<>(d2, 2), result.get(1));
    Assert.assertEquals(new Tuple<>(d3, 3), result.get(2));
    Assert.assertEquals(new Tuple<>(d4, 3), result.get(3));
    Assert.assertEquals(new Tuple<>(w1, 0), result.get(4));
}

Example 10 with Tuple

use of org.apache.reef.io.Tuple in project mist by snuspl.

the class QueryInfoStoreTest method diskStoreTest.

/**
 * Tests whether the PlanStore correctly saves, deletes and loads the operator chain dag.
 * @throws InjectionException
 * @throws IOException
 */
@Test(timeout = 1000)
public void diskStoreTest() throws InjectionException, IOException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Jar files
    final List<ByteBuffer> jarFiles = new LinkedList<>();
    final ByteBuffer byteBuffer1 = ByteBuffer.wrap(new byte[] { 0, 1, 0, 1, 1, 1 });
    jarFiles.add(byteBuffer1);
    final Injector injector = Tang.Factory.getTang().newInjector();
    final QueryInfoStore store = injector.getInstance(QueryInfoStore.class);
    final ApplicationCodeManager applicationCodeManager = injector.getInstance(ApplicationCodeManager.class);
    final ApplicationMap applicationMap = injector.getInstance(ApplicationMap.class);
    final String queryId1 = "testQuery1";
    final String queryId2 = "testQuery2";
    final String tmpFolderPath = injector.getNamedInstance(SharedStorePath.class);
    final File folder = new File(tmpFolderPath);
    // Store jar files
    final List<String> paths = applicationCodeManager.registerNewAppCode(jarFiles).getJarPaths();
    for (int i = 0; i < jarFiles.size(); i++) {
        final ByteBuffer buf = ByteBuffer.allocateDirect(jarFiles.get(i).capacity());
        final String path = paths.get(i);
        final FileInputStream fis = new FileInputStream(path);
        final FileChannel channel = fis.getChannel();
        channel.read(buf);
        Assert.assertEquals(jarFiles.get(i), buf);
    }
    final ApplicationInfo applicationInfo = mock(ApplicationInfo.class);
    when(applicationInfo.getApplicationId()).thenReturn(TestParameters.SUPER_GROUP_ID);
    when(applicationInfo.getJarFilePath()).thenReturn(paths);
    applicationMap.putIfAbsent(TestParameters.SUPER_GROUP_ID, applicationInfo);
    // Generate logical plan
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroDag1 = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(paths).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final AvroDag avroDag2 = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(paths).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    // Store the chained dag
    store.saveAvroDag(new Tuple<>(queryId1, avroDag1));
    store.saveAvroDag(new Tuple<>(queryId2, avroDag2));
    while (!(store.isStored(queryId1) && store.isStored(queryId2))) {
    // Wait until the plan is stored
    }
    Assert.assertTrue(new File(tmpFolderPath, queryId1 + ".plan").exists());
    Assert.assertTrue(new File(tmpFolderPath, queryId2 + ".plan").exists());
    // Test stored file
    final AvroDag loadedDag1 = store.load(queryId1);
    Assert.assertEquals(avroDag1.getEdges(), loadedDag1.getEdges());
    Assert.assertEquals(avroDag1.getSchema(), loadedDag1.getSchema());
    testVerticesEqual(avroDag1.getAvroVertices(), loadedDag1.getAvroVertices());
    final AvroDag loadedDag2 = store.load(queryId2);
    Assert.assertEquals(avroDag2.getEdges(), loadedDag2.getEdges());
    Assert.assertEquals(avroDag2.getSchema(), loadedDag2.getSchema());
    testVerticesEqual(avroDag2.getAvroVertices(), loadedDag2.getAvroVertices());
    // Test deletion
    store.delete(queryId1);
    store.delete(queryId2);
    Assert.assertFalse(store.isStored(queryId1));
    Assert.assertFalse(new File(tmpFolderPath, queryId1 + ".plan").exists());
    Assert.assertFalse(store.isStored(queryId2));
    Assert.assertFalse(new File(tmpFolderPath, queryId2 + ".plan").exists());
    for (final String path : paths) {
        Assert.assertFalse(new File(path).exists());
    }
    folder.delete();
}