Examples with Tuple2 edu.snu.mist.common.types.Tuple2 used on opensource projects

Example 11 with Tuple2

use of edu.snu.mist.common.types.Tuple2 in project mist by snuspl.

the class DefaultDagGeneratorImplTest method testPlanGenerator.

/**
 * Round-trip test of de-serializing AvroOperatorChainDag.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Generate avro operator chain dag
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
    final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
    final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
    final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
    final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
    // Test execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    final Set<ExecutionVertex> sources = dag.getRootVertices();
    Assert.assertEquals(1, sources.size());
    Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
    final PhysicalSource source = (PhysicalSource) sources.iterator().next();
    final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
    Assert.assertEquals(1, nextOps.size());
    final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
    final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
    final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
    final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
    final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
    Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
    Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
    Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
    Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
    Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}

Example 12 with Tuple2

use of edu.snu.mist.common.types.Tuple2 in project mist by snuspl.

the class MISTDefaultExecutionEnvironmentImplTest method testMISTDefaultExecutionEnvironment.

/**
 * This unit test creates mock jar file, mocking driver, and mocking task and tests
 * whether a test query can be serialized and sent via MISTDefaultExecutionEnvironmentImpl.
 */
@Test
public void testMISTDefaultExecutionEnvironment() throws IOException {
    // Step 1: Launch mock RPC Server
    final Server masterServer = new NettyServer(new SpecificResponder(ClientToMasterMessage.class, new MockMasterServer(host, taskPortNum)), new InetSocketAddress(masterPortNum));
    final Server taskServer = new NettyServer(new SpecificResponder(ClientToTaskMessage.class, new MockTaskServer(testQueryResult)), new InetSocketAddress(taskPortNum));
    // Step 2: Upload jar file
    final int suffixStartIndex = mockJarOutName.lastIndexOf(".");
    final String mockJarOutPrefix = mockJarOutName.substring(0, suffixStartIndex);
    final String mockJarOutSuffix = mockJarOutName.substring(suffixStartIndex);
    final List<String> jarPaths = new LinkedList<>();
    final Path tempJarFile = Files.createTempFile(mockJarOutPrefix, mockJarOutSuffix);
    jarPaths.add(tempJarFile.toString());
    final MISTExecutionEnvironment executionEnvironment = new MISTDefaultExecutionEnvironmentImpl(host, masterPortNum);
    final JarUploadResult jarUploadResult = executionEnvironment.submitJar(jarPaths);
    Assert.assertEquals("app_id", jarUploadResult.getIdentifier());
    // Step 3: Generate a new query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(jarUploadResult.getIdentifier()).socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput("localhost", 13667);
    final MISTQuery query = queryBuilder.build();
    System.err.println(mockJarOutPrefix);
    System.err.println(mockJarOutSuffix);
    // Step 4: Send a query and check whether the query comes to the task correctly
    final APIQueryControlResult result = executionEnvironment.submitQuery(query);
    Assert.assertEquals(result.getQueryId(), testQueryResult);
    masterServer.close();
    taskServer.close();
    Files.delete(tempJarFile);
}

Example 13 with Tuple2

use of edu.snu.mist.common.types.Tuple2 in project mist by snuspl.

the class MISTQueryTest method mistComplexQuerySerializeTest.

/**
 * This method tests a serialization of a complex query, containing 9 vertices.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void mistComplexQuerySerializeTest() throws InjectionException, IOException, URISyntaxException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final ContinuousStream<String> sourceStream = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF, TestParameters.PUNCTUATED_WATERMARK_CONF);
    final ContinuousStream<String> flatMapStream = sourceStream.flatMap(expectedFlatMapFunc);
    final ContinuousStream<String> filterStream = flatMapStream.filter(expectedFilterPredicate);
    final ContinuousStream<Tuple2<String, Integer>> mapStream = filterStream.map(expectedMapFunc);
    final WindowedStream<Tuple2<String, Integer>> windowedStream = mapStream.window(new TimeWindowInformation(expectedWindowSize, expectedWindowEmissionInterval));
    final ContinuousStream<Map<String, Integer>> reduceByKeyStream = windowedStream.reduceByKeyWindow(0, String.class, expectedReduceFunc);
    final MISTStream<String> sinkStream = reduceByKeyStream.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    // Build a query
    final MISTQuery complexQuery = queryBuilder.build();
    final Tuple<List<AvroVertex>, List<Edge>> serializedDAG = complexQuery.getAvroOperatorDag();
    final List<AvroVertex> vertices = serializedDAG.getKey();
    Assert.assertEquals(7, vertices.size());
    Assert.assertEquals(sourceStream.getConfiguration(), vertices.get(0).getConfiguration());
    Assert.assertEquals(flatMapStream.getConfiguration(), vertices.get(1).getConfiguration());
    Assert.assertEquals(filterStream.getConfiguration(), vertices.get(2).getConfiguration());
    Assert.assertEquals(mapStream.getConfiguration(), vertices.get(3).getConfiguration());
    Assert.assertEquals(windowedStream.getConfiguration(), vertices.get(4).getConfiguration());
    Assert.assertEquals(reduceByKeyStream.getConfiguration(), vertices.get(5).getConfiguration());
    Assert.assertEquals(sinkStream.getConfiguration(), vertices.get(6).getConfiguration());
    final List<Edge> edges = serializedDAG.getValue();
    final List<Edge> expectedEdges = Arrays.asList(Edge.newBuilder().setFrom(0).setTo(1).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(1).setTo(2).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(2).setTo(3).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(3).setTo(4).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(4).setTo(5).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(5).setTo(6).setDirection(Direction.LEFT).setBranchIndex(0).build());
    Assert.assertEquals(new HashSet<>(expectedEdges), new HashSet<>(edges));
}

Example 14 with Tuple2

use of edu.snu.mist.common.types.Tuple2 in project mist by snuspl.

the class ContinuousStreamTest method testTimeWindowedStream.

/**
 * Test for creating time-based WindowedStream from ContinuousStream.
 */
@Test
public void testTimeWindowedStream() {
    final WindowedStream<Tuple2<String, Integer>> timeWindowedStream = filteredMappedStream.window(new TimeWindowInformation(windowSize, windowEmissionInterval));
    final Map<String, String> conf = timeWindowedStream.getConfiguration();
    checkSizeBasedWindowInfo(windowSize, windowEmissionInterval, conf);
    // Check map -> timeWindow
    checkEdges(queryBuilder.build().getDAG(), 1, filteredMappedStream, timeWindowedStream, new MISTEdge(Direction.LEFT));
}

Example 15 with Tuple2

use of edu.snu.mist.common.types.Tuple2 in project mist by snuspl.

the class StateTransitionOperatorTest method testStateTransitionOperator.

/**
 * Test StateTransitionOperator.
 * Final states are "1" and "4".
 */
@Test
public void testStateTransitionOperator() throws InterruptedException {
    // input events
    // expected state transitions: 0 -- 1 -- 0 -- 3 -- 4, "1" and "4" are emitted.
    final Map<String, Integer> value1 = new HashMap<>();
    value1.put("number", 1);
    final MistDataEvent data1 = new MistDataEvent(value1, 0L);
    final Map<String, Integer> value2 = new HashMap<>();
    value2.put("number", 2);
    final MistDataEvent data2 = new MistDataEvent(value2, 1L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(7L);
    final Map<String, Integer> value3 = new HashMap<>();
    value3.put("number", 3);
    final MistDataEvent data3 = new MistDataEvent(value3, 10L);
    final Map<String, Integer> value4 = new HashMap<>();
    value4.put("number", 4);
    final MistDataEvent data4 = new MistDataEvent(value4, 11L);
    // make a set of final states
    final Set<String> finalSet = new HashSet<>();
    finalSet.add("1");
    finalSet.add("4");
    // make a state table
    final Map<String, Collection<Tuple2<MISTPredicate, String>>> stateTable = new HashMap<>();
    final Collection<Tuple2<MISTPredicate, String>> list0 = new ArrayList<>();
    list0.add(new Tuple2<>(new RuleBasedEQPredicate("number", 1), "1"));
    list0.add(new Tuple2<>(new RuleBasedEQPredicate("number", 3), "3"));
    final Collection<Tuple2<MISTPredicate, String>> list1 = new ArrayList<>();
    list1.add(new Tuple2<>(new RuleBasedEQPredicate("number", 2), "0"));
    final Collection<Tuple2<MISTPredicate, String>> list3 = new ArrayList<>();
    list3.add(new Tuple2<>(new RuleBasedEQPredicate("number", 4), "4"));
    stateTable.put("0", list0);
    stateTable.put("1", list1);
    stateTable.put("3", list3);
    // make a state transition operator
    final StateTransitionOperator stateTransitionOperator = new StateTransitionOperator("0", finalSet, stateTable);
    final List<MistEvent> result = new LinkedList<>();
    stateTransitionOperator.setOutputEmitter(new OutputBufferEmitter(result));
    stateTransitionOperator.processLeftData(data1);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(data1, result.get(0));
    Assert.assertEquals("1", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftData(data2);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals("0", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
    Assert.assertEquals("0", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftData(data3);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals("3", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftData(data4);
    Assert.assertEquals(3, result.size());
    Assert.assertEquals(data4, result.get(2));
    Assert.assertEquals("4", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
}