Examples with MISTQuery edu.snu.mist.client.MISTQuery used on opensource projects

Example 1 with MISTQuery

use of edu.snu.mist.client.MISTQuery in project mist by snuspl.

the class ContinuousStreamTest method testBasicOperatorStream.

/**
 * Test for basic stateless OperatorStreams.
 */
@Test
public void testBasicOperatorStream() throws IOException {
    final Map<String, String> filteredConf = filteredMappedStream.getConfiguration();
    Assert.assertEquals(SerializeUtils.serializeToString(defaultMap), filteredConf.get(ConfKeys.OperatorConf.UDF_STRING.name()));
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    // Check src -> filiter
    final Map<MISTStream, MISTEdge> neighbors = dag.getEdges(sourceStream);
    Assert.assertEquals(1, neighbors.size());
    final MISTEdge edgeInfo = neighbors.get(filteredStream);
    Assert.assertEquals(Direction.LEFT, edgeInfo.getDirection());
    Assert.assertEquals(0, edgeInfo.getIndex());
    // Check filter -> map
    final Map<MISTStream, MISTEdge> neighbors2 = dag.getEdges(filteredStream);
    Assert.assertEquals(1, neighbors2.size());
    final MISTEdge edgeInfo2 = neighbors2.get(filteredMappedStream);
    Assert.assertEquals(Direction.LEFT, edgeInfo2.getDirection());
    Assert.assertEquals(0, edgeInfo2.getIndex());
}

Example 2 with MISTQuery

use of edu.snu.mist.client.MISTQuery in project mist by snuspl.

the class ContinuousStreamTest method testJoinOperatorStream.

/**
 * Test for join operation.
 */
@Test
public void testJoinOperatorStream() throws InjectionException, IOException, ClassNotFoundException {
    final ContinuousStream<String> firstInputStream = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> secondInputStream = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final MISTBiPredicate<String, String> joinBiPred = (string1, string2) -> string1.equals(string2);
    final WindowedStream<Tuple2<String, String>> joinedStream = firstInputStream.join(secondInputStream, joinBiPred, new CountWindowInformation(5, 3));
    final Map<String, String> conf = joinedStream.getConfiguration();
    Assert.assertEquals(SerializeUtils.serializeToString(joinBiPred), conf.get(ConfKeys.OperatorConf.UDF_STRING.name()));
    // Check first input -> mapped
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final MISTStream firstMappedInputStream = getNextOperatorStream(dag, 1, firstInputStream, new MISTEdge(Direction.LEFT));
    // Check second input -> mapped
    final MISTStream secondMappedInputStream = getNextOperatorStream(dag, 1, secondInputStream, new MISTEdge(Direction.LEFT));
    // Check two mapped input -> unified
    final MISTStream firstUnifiedStream = getNextOperatorStream(dag, 1, firstMappedInputStream, new MISTEdge(Direction.LEFT));
    final MISTStream secondUnifiedStream = getNextOperatorStream(dag, 1, secondMappedInputStream, new MISTEdge(Direction.RIGHT));
    Assert.assertEquals(firstUnifiedStream, secondUnifiedStream);
    // Check unified stream -> windowed
    final MISTStream windowedStream = getNextOperatorStream(dag, 1, firstUnifiedStream, new MISTEdge(Direction.LEFT));
    // Check windowed stream -> joined
    checkEdges(dag, 1, windowedStream, joinedStream, new MISTEdge(Direction.LEFT));
}

Example 3 with MISTQuery

use of edu.snu.mist.client.MISTQuery in project mist by snuspl.

the class ContinuousStreamTest method testUnionOperatorStream.

/**
 * Test for union operator.
 */
@Test
public void testUnionOperatorStream() {
    final ContinuousStream<Tuple2<String, Integer>> filteredMappedStream2 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).filter(s -> s.contains("A")).map(s -> new Tuple2<>(s, 1));
    final ContinuousStream<Tuple2<String, Integer>> unifiedStream = filteredMappedStream.union(filteredMappedStream2);
    // Check filteredMappedStream (LEFT)  ---> union
    // filteredMappedStream2 (RIGHT) --/
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final Map<MISTStream, MISTEdge> n1 = dag.getEdges(filteredMappedStream);
    final Map<MISTStream, MISTEdge> n2 = dag.getEdges(filteredMappedStream2);
    Assert.assertEquals(1, n1.size());
    Assert.assertEquals(1, n2.size());
    Assert.assertEquals(new MISTEdge(Direction.LEFT), n1.get(unifiedStream));
    Assert.assertEquals(new MISTEdge(Direction.RIGHT), n2.get(unifiedStream));
}

Example 4 with MISTQuery

use of edu.snu.mist.client.MISTQuery in project mist by snuspl.

the class DefaultDagGeneratorImplTest method testPlanGenerator.

/**
 * Round-trip test of de-serializing AvroOperatorChainDag.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Generate avro operator chain dag
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
    final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
    final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
    final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
    final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
    // Test execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    final Set<ExecutionVertex> sources = dag.getRootVertices();
    Assert.assertEquals(1, sources.size());
    Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
    final PhysicalSource source = (PhysicalSource) sources.iterator().next();
    final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
    Assert.assertEquals(1, nextOps.size());
    final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
    final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
    final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
    final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
    final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
    Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
    Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
    Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
    Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
    Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}

Example 5 with MISTQuery

use of edu.snu.mist.client.MISTQuery in project mist by snuspl.

the class GroupRecoveryTest method testSingleQueryRecovery.

@Test(timeout = 500000)
public void testSingleQueryRecovery() throws Exception {
    // Start source servers.
    final CountDownLatch sourceCountDownLatch1 = new CountDownLatch(1);
    final NettyTextMessageStreamGenerator textMessageStreamGenerator1 = new NettyTextMessageStreamGenerator(SERVER_ADDR, SOURCE_PORT1, new TestChannelHandler(sourceCountDownLatch1));
    final TestSinkHandler handler1 = new TestSinkHandler();
    final NettyTextMessageOutputReceiver receiver1 = new NettyTextMessageOutputReceiver("localhost", SINK_PORT, handler1);
    // Submit query.
    final MISTQuery query = buildQuery();
    // Generate avro chained dag : needed parts from MISTExamplesUtils.submitQuery()
    final Tuple<List<AvroVertex>, List<Edge>> initialAvroOpChainDag = query.getAvroOperatorDag();
    final String appId = "testApp";
    final String queryId = "testQuery";
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroDag = avroDagBuilder.setAppId(appId).setQueryId(queryId).setJarPaths(new ArrayList<>()).setAvroVertices(initialAvroOpChainDag.getKey()).setEdges(initialAvroOpChainDag.getValue()).build();
    // Build QueryManager.
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(PeriodicCheckpointPeriod.class, "1000");
    jcb.bindImplementation(QueryManager.class, GroupAwareQueryManagerImpl.class);
    jcb.bindImplementation(QueryStarter.class, ImmediateQueryMergingStarter.class);
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    injector.bindVolatileInstance(GroupIdRequestor.class, new TestGroupIdRequestor());
    injector.bindVolatileInstance(TaskStatsUpdater.class, mock(TaskStatsUpdater.class));
    final CheckpointManager checkpointManager = injector.getInstance(CheckpointManager.class);
    final QueryManager queryManager = injector.getInstance(QueryManager.class);
    queryManager.createApplication(appId, Arrays.asList(""));
    queryManager.create(avroDag);
    // Wait until all sources connect to stream generator
    sourceCountDownLatch1.await();
    final ExecutionDags executionDags = checkpointManager.getApplication(appId).getGroups().get(0).getExecutionDags();
    Assert.assertEquals(executionDags.values().size(), 1);
    final ExecutionDag executionDag = executionDags.values().iterator().next();
    // 1st stage. Push inputs to all sources and see if the results are proper.
    SRC0INPUT1.forEach(textMessageStreamGenerator1::write);
    LATCH1.await();
    Assert.assertEquals("{aa=2, bb=1, cc=1}", handler1.getResults().get(handler1.getResults().size() - 1));
    // Sleep 2 seconds for the checkpoint events to be sent out.
    sleep(2000);
    // Checkpoint the entire MISTTask, delete it, and restore it to see if it works.
    final String groupId = checkpointManager.getApplication(appId).getGroups().get(0).getGroupId();
    checkpointManager.checkpointGroup(groupId);
    checkpointManager.deleteGroup(groupId);
    // Close the generator.
    textMessageStreamGenerator1.close();
    receiver1.close();
    // Restart source servers.
    final CountDownLatch sourceCountDownLatch2 = new CountDownLatch(1);
    final NettyTextMessageStreamGenerator textMessageStreamGenerator2 = new NettyTextMessageStreamGenerator(SERVER_ADDR, SOURCE_PORT1, new TestChannelHandler(sourceCountDownLatch2));
    final TestSinkHandler handler2 = new TestSinkHandler();
    final NettyTextMessageOutputReceiver receiver2 = new NettyTextMessageOutputReceiver("localhost", SINK_PORT, handler2);
    // Recover the group.
    checkpointManager.recoverGroup(groupId);
    // Wait until all sources connect to stream generator
    sourceCountDownLatch2.await();
    final ExecutionDags executionDags2 = checkpointManager.getApplication(appId).getGroups().get(0).getExecutionDags();
    Assert.assertEquals(executionDags2.values().size(), 1);
    final ExecutionDag executionDag2 = executionDags2.values().iterator().next();
    // 2nd stage. Push inputs to the recovered the query to see if it works.
    SRC0INPUT2.forEach(textMessageStreamGenerator2::write);
    LATCH2.await();
    Assert.assertEquals("{aa=3, bb=3, cc=3}", handler2.getResults().get(handler2.getResults().size() - 1));
    // Close the generators.
    textMessageStreamGenerator2.close();
    receiver2.close();
}