Examples with DAG edu.snu.mist.common.graph.DAG used on opensource projects

Example 1 with DAG

use of edu.snu.mist.common.graph.DAG in project mist by snuspl.

the class ContinuousStreamTest method testJoinOperatorStream.

/**
 * Test for join operation.
 */
@Test
public void testJoinOperatorStream() throws InjectionException, IOException, ClassNotFoundException {
    final ContinuousStream<String> firstInputStream = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> secondInputStream = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final MISTBiPredicate<String, String> joinBiPred = (string1, string2) -> string1.equals(string2);
    final WindowedStream<Tuple2<String, String>> joinedStream = firstInputStream.join(secondInputStream, joinBiPred, new CountWindowInformation(5, 3));
    final Map<String, String> conf = joinedStream.getConfiguration();
    Assert.assertEquals(SerializeUtils.serializeToString(joinBiPred), conf.get(ConfKeys.OperatorConf.UDF_STRING.name()));
    // Check first input -> mapped
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final MISTStream firstMappedInputStream = getNextOperatorStream(dag, 1, firstInputStream, new MISTEdge(Direction.LEFT));
    // Check second input -> mapped
    final MISTStream secondMappedInputStream = getNextOperatorStream(dag, 1, secondInputStream, new MISTEdge(Direction.LEFT));
    // Check two mapped input -> unified
    final MISTStream firstUnifiedStream = getNextOperatorStream(dag, 1, firstMappedInputStream, new MISTEdge(Direction.LEFT));
    final MISTStream secondUnifiedStream = getNextOperatorStream(dag, 1, secondMappedInputStream, new MISTEdge(Direction.RIGHT));
    Assert.assertEquals(firstUnifiedStream, secondUnifiedStream);
    // Check unified stream -> windowed
    final MISTStream windowedStream = getNextOperatorStream(dag, 1, firstUnifiedStream, new MISTEdge(Direction.LEFT));
    // Check windowed stream -> joined
    checkEdges(dag, 1, windowedStream, joinedStream, new MISTEdge(Direction.LEFT));
}

Example 2 with DAG

use of edu.snu.mist.common.graph.DAG in project mist by snuspl.

the class ContinuousStreamTest method testUnionOperatorStream.

/**
 * Test for union operator.
 */
@Test
public void testUnionOperatorStream() {
    final ContinuousStream<Tuple2<String, Integer>> filteredMappedStream2 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).filter(s -> s.contains("A")).map(s -> new Tuple2<>(s, 1));
    final ContinuousStream<Tuple2<String, Integer>> unifiedStream = filteredMappedStream.union(filteredMappedStream2);
    // Check filteredMappedStream (LEFT)  ---> union
    // filteredMappedStream2 (RIGHT) --/
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final Map<MISTStream, MISTEdge> n1 = dag.getEdges(filteredMappedStream);
    final Map<MISTStream, MISTEdge> n2 = dag.getEdges(filteredMappedStream2);
    Assert.assertEquals(1, n1.size());
    Assert.assertEquals(1, n2.size());
    Assert.assertEquals(new MISTEdge(Direction.LEFT), n1.get(unifiedStream));
    Assert.assertEquals(new MISTEdge(Direction.RIGHT), n2.get(unifiedStream));
}

Example 3 with DAG

use of edu.snu.mist.common.graph.DAG in project mist by snuspl.

the class DefaultDagGeneratorImplTest method testPlanGenerator.

/**
 * Round-trip test of de-serializing AvroOperatorChainDag.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Generate avro operator chain dag
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
    final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
    final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
    final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
    final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
    // Test execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    final Set<ExecutionVertex> sources = dag.getRootVertices();
    Assert.assertEquals(1, sources.size());
    Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
    final PhysicalSource source = (PhysicalSource) sources.iterator().next();
    final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
    Assert.assertEquals(1, nextOps.size());
    final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
    final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
    final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
    final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
    final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
    Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
    Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
    Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
    Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
    Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}

Example 4 with DAG

use of edu.snu.mist.common.graph.DAG in project mist by snuspl.

the class ImmediateQueryMergingStarterTest method mergingSameSourceAndSameOperatorQueriesOneGroupTest.

/**
 * Case 3: Merging two dags that have same source and operator chain.
 * @throws InjectionException
 */
@Test
public void mergingSameSourceAndSameOperatorQueriesOneGroupTest() throws InjectionException, IOException, ClassNotFoundException {
    // Create a query 1:
    // src1 -> oc1 -> sink1
    final List<String> result1 = new LinkedList<>();
    final Map<String, String> sourceConf = idAndConfGenerator.generateConf();
    final Map<String, String> operatorConf = idAndConfGenerator.generateConf();
    final TestSource src1 = generateSource(sourceConf);
    final Map<String, String> sinkConf1 = idAndConfGenerator.generateConf();
    final PhysicalOperator physicalOp1 = generateFilterOperator(operatorConf, (s) -> true);
    final PhysicalSink<String> sink1 = generateSink(sinkConf1, result1);
    // Config vertices
    final ConfigVertex srcVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.OPERATOR, operatorConf);
    final ConfigVertex sinkVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf1);
    final List<String> paths1 = mock(List.class);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple1 = generateSimpleDag(src1, physicalOp1, sink1, srcVertex1, ocVertex1, sinkVertex1);
    // Create a query 2:
    // src2 -> oc2 -> sink2
    // The configuration of src2 and operatorChain2 is same as that of src1 and operatorChain2.
    final List<String> result2 = new LinkedList<>();
    final Map<String, String> sinkConf2 = idAndConfGenerator.generateConf();
    final TestSource src2 = generateSource(sourceConf);
    final PhysicalOperator physicalOp2 = generateFilterOperator(operatorConf, (s) -> true);
    final PhysicalSink<String> sink2 = generateSink(sinkConf2, result2);
    final List<String> paths2 = mock(List.class);
    // Config vertices
    final ConfigVertex srcVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.OPERATOR, operatorConf);
    final ConfigVertex sinkVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf2);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple2 = generateSimpleDag(src2, physicalOp2, sink2, srcVertex2, ocVertex2, sinkVertex2);
    // Execute two queries
    final Query query1 = mock(Query.class);
    final Query query2 = mock(Query.class);
    final String query1Id = "q1";
    final String query2Id = "q2";
    queryStarter.start(query1Id, query1, dagTuple1.getKey(), paths1);
    queryStarter.start(query2Id, query2, dagTuple2.getKey(), paths2);
    // Generate events for the merged query and check if the dag is executed correctly
    final String data = "Hello";
    src1.send(data);
    Assert.assertEquals(1, src1.getSourceOutputEmitter().numberOfEvents());
    // This is not created because the source is the same!
    Assert.assertEquals(null, src2.getSourceOutputEmitter());
    Assert.assertEquals(1, src1.getSourceOutputEmitter().processAllEvent());
    Assert.assertEquals(Arrays.asList(data), result1);
    Assert.assertEquals(Arrays.asList(data), result2);
    // Src2 and 1 are the same, so the output emitter of src2 should be null
    try {
        src2.send(data);
        Assert.fail("OutputEmitter should be null");
    } catch (final NullPointerException e) {
    // do nothing
    }
    // Check queryIdConfigDagMap
    Assert.assertEquals(dagTuple1.getKey(), queryIdConfigDagMap.get(query1Id));
    Assert.assertEquals(dagTuple2.getKey(), queryIdConfigDagMap.get(query2Id));
    // Check execution dags
    final Collection<ExecutionDag> expectedDags = new HashSet<>();
    final DAG<ExecutionVertex, MISTEdge> mergedDag = dagTuple1.getValue().getDag();
    mergedDag.addVertex(sink2);
    mergedDag.addEdge(physicalOp1, sink2, new MISTEdge(Direction.LEFT));
    expectedDags.add(srcAndDagMap.get(sourceConf));
    Assert.assertEquals(expectedDags, executionDags.values());
    // Check srcAndDagMap
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, srcAndDagMap.get(sourceConf).getDag()));
    // Check executionVertexDagMap
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(src1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(physicalOp1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(sink1).getDag()));
    // They are merged, so src2, oc2 and sink2 should be included in dag1
    Assert.assertNull(executionVertexDagMap.get(src2));
    Assert.assertNull(executionVertexDagMap.get(physicalOp2));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(sink2).getDag()));
    // Check configExecutionVertexMap
    Assert.assertEquals(src1, configExecutionVertexMap.get(srcVertex1));
    Assert.assertEquals(physicalOp1, configExecutionVertexMap.get(ocVertex1));
    Assert.assertEquals(sink1, configExecutionVertexMap.get(sinkVertex1));
    Assert.assertEquals(src1, configExecutionVertexMap.get(srcVertex2));
    Assert.assertEquals(physicalOp1, configExecutionVertexMap.get(ocVertex2));
    Assert.assertEquals(sink2, configExecutionVertexMap.get(sinkVertex2));
    // Check reference count
    Assert.assertEquals(2, (int) executionVertexCountMap.get(src1));
    Assert.assertEquals(2, (int) executionVertexCountMap.get(physicalOp1));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(sink1));
    Assert.assertNull(executionVertexCountMap.get(src2));
    Assert.assertNull(executionVertexCountMap.get(physicalOp2));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(sink2));
}

Example 5 with DAG

use of edu.snu.mist.common.graph.DAG in project mist by snuspl.

the class QueryManagerTest method testSubmitComplexQueryHelper.

@SuppressWarnings("unchecked")
private void testSubmitComplexQueryHelper(final Configuration conf) throws Exception {
    final String queryId = "testQuery";
    final List<String> inputs = Arrays.asList("mist is a cloud of tiny water droplets suspended in the atmosphere", "a mist rose out of the river", "the peaks were shrouded in mist");
    // Expected results
    final List<Map<String, Integer>> intermediateResult = getIntermediateResult(inputs);
    final List<String> expectedSink1Output = intermediateResult.stream().map(input -> input.toString()).collect(Collectors.toList());
    final List<Integer> expectedSink2Output = intermediateResult.stream().map(totalCountMapFunc).collect(Collectors.toList());
    // Number of expected outputs
    final CountDownLatch countDownAllOutputs = new CountDownLatch(intermediateResult.size() * 2);
    // Create the execution DAG of the query
    final ExecutionDag executionDag = new ExecutionDag(new AdjacentListDAG<>());
    // Create source
    final TestDataGenerator dataGenerator = new TestDataGenerator(inputs);
    final EventGenerator eventGenerator = new PunctuatedEventGenerator(null, input -> false, null, 0, null, null);
    final PhysicalSource src = new PhysicalSourceImpl("testSource", new HashMap<>(), dataGenerator, eventGenerator);
    final Injector injector = Tang.Factory.getTang().newInjector(conf);
    // Create sinks
    final List<String> sink1Result = new LinkedList<>();
    final List<Integer> sink2Result = new LinkedList<>();
    final PhysicalSink sink1 = new PhysicalSinkImpl<>("sink1", null, new TestWithCountDownSink<String>(sink1Result, countDownAllOutputs));
    final PhysicalSink sink2 = new PhysicalSinkImpl<>("sink2", null, new TestWithCountDownSink<Integer>(sink2Result, countDownAllOutputs));
    // Fake operator chain dag of QueryManager
    final AvroDag fakeAvroDag = new AvroDag();
    fakeAvroDag.setQueryId(queryId);
    // fakeAvroDag.setSuperGroupId("testGroup");
    // Construct execution dag
    constructExecutionDag(executionDag, src, sink1, sink2);
    // Create mock DagGenerator. It returns the above  execution dag
    final ConfigDagGenerator configDagGenerator = mock(ConfigDagGenerator.class);
    final DAG<ConfigVertex, MISTEdge> configDag = mock(DAG.class);
    when(configDagGenerator.generate(fakeAvroDag)).thenReturn(configDag);
    final DagGenerator dagGenerator = mock(DagGenerator.class);
    // when(dagGenerator.generate(configDag, tuple.getValue().getJarFilePaths())).thenReturn(executionDag);
    // Build QueryManager
    final QueryManager queryManager = queryManagerBuild(fakeAvroDag, configDagGenerator, dagGenerator, injector);
    queryManager.create(fakeAvroDag);
    // Wait until all of the outputs are generated
    countDownAllOutputs.await();
    // Check the outputs
    Assert.assertEquals(expectedSink1Output, sink1Result);
    Assert.assertEquals(expectedSink2Output, sink2Result);
    src.close();
    queryManager.close();
    // Delete plan directory and plans
    deletePlans(injector);
}