Examples with MISTEdge edu.snu.mist.common.graph.MISTEdge used on opensource projects

Example 21 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class OperatorChainDagGeneratorTest method testBranchTest.

/**
 * Test branch chaining.
 * logical dag:
 *                      -> op14 -> sink2
 * src1 -> op11 -> op12 -> op13 -> sink1
 *                      -> op15 -> sink3
 * should be converted to the expected chained dag:
 *                        -> [op14] -> sink2
 * src1 -> [op11 -> op12] -> [op13] -> sink1
 *                        -> [op15] -> sink3
 */
@Test
public void testBranchTest() throws InjectionException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op11 = src1.filter((x) -> true);
    final ContinuousStream<String> op12 = op11.filter((x) -> true);
    final ContinuousStream<String> op13 = op12.filter((x) -> true);
    final ContinuousStream<String> op14 = op12.filter((x) -> true);
    final ContinuousStream<String> op15 = op12.filter((x) -> true);
    final MISTStream<String> sink1 = op13.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink2 = op14.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink3 = op15.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final OperatorChainDagGenerator chainDagGenerator = new OperatorChainDagGenerator(dag);
    final DAG<List<MISTStream>, MISTEdge> chainPlan = chainDagGenerator.generateOperatorChainDAG();
    // Expected outputs
    final List<MISTStream> src1List = Arrays.asList(src1);
    final List<MISTStream> op11op12 = Arrays.asList(op11, op12);
    final List<MISTStream> op13List = Arrays.asList(op13);
    final List<MISTStream> op14List = Arrays.asList(op14);
    final List<MISTStream> op15List = Arrays.asList(op15);
    final List<MISTStream> sink1List = Arrays.asList(sink1);
    final List<MISTStream> sink2List = Arrays.asList(sink2);
    final List<MISTStream> sink3List = Arrays.asList(sink3);
    // Check src1 -> [op11->op12] edge
    final Map<List<MISTStream>, MISTEdge> e1 = chainPlan.getEdges(src1List);
    final Map<List<MISTStream>, MISTEdge> result1 = new HashMap<>();
    result1.put(op11op12, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check [op11->op12] -> [op13] edges
    // -> [op14]
    // -> [op15]
    final Map<List<MISTStream>, MISTEdge> e2 = chainPlan.getEdges(op11op12);
    final Map<List<MISTStream>, MISTEdge> result2 = new HashMap<>();
    result2.put(op13List, new MISTEdge(Direction.LEFT));
    result2.put(op14List, new MISTEdge(Direction.LEFT));
    result2.put(op15List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e2, result2);
    // Check [op14] -> [sink2] edge
    final Map<List<MISTStream>, MISTEdge> e3 = chainPlan.getEdges(op14List);
    final Map<List<MISTStream>, MISTEdge> result3 = new HashMap<>();
    result3.put(sink2List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e3, result3);
    // Check [op13] -> [sink1] edge
    final Map<List<MISTStream>, MISTEdge> e4 = chainPlan.getEdges(op13List);
    final Map<List<MISTStream>, MISTEdge> result4 = new HashMap<>();
    result4.put(sink1List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e4, result4);
    // Check [op15] -> [sink3] edge
    final Map<List<MISTStream>, MISTEdge> e5 = chainPlan.getEdges(op15List);
    final Map<List<MISTStream>, MISTEdge> result5 = new HashMap<>();
    result5.put(sink3List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e5, result5);
}

Example 22 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class OperatorChainDagGeneratorTest method testMergingQueryPartitioning.

/**
 * Test merge chaining.
 * logical dag:
 * src1 -> op11 -> op12 ->
 * src2 ---------> op21 -> op13 -> op14 -> sink1
 * src3 -----------------> op31 ->
 *
 * should be converted to the expected operator chain dag:
 * src1 -> [op11 -> op12] ->
 * src2 ---------> [op21] -> [op13] -> [op14] -> sink1
 * src3 -------------------> [op31] ->
 */
@Test
public void testMergingQueryPartitioning() throws InjectionException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> src2 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> src3 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op11 = src1.filter((x) -> true);
    final ContinuousStream<String> op12 = op11.filter((x) -> true);
    final ContinuousStream<String> op21 = src2.filter((x) -> true);
    final ContinuousStream<String> op13 = op12.union(op21);
    final ContinuousStream<String> op31 = src3.filter((x) -> true);
    final ContinuousStream<String> op14 = op13.union(op31);
    final MISTStream<String> sink1 = op14.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final OperatorChainDagGenerator chainDagGenerator = new OperatorChainDagGenerator(dag);
    final DAG<List<MISTStream>, MISTEdge> operatorChainDag = chainDagGenerator.generateOperatorChainDAG();
    // Expected outputs
    final List<MISTStream> src1List = Arrays.asList(src1);
    final List<MISTStream> src2List = Arrays.asList(src2);
    final List<MISTStream> src3List = Arrays.asList(src3);
    final List<MISTStream> op11op12 = Arrays.asList(op11, op12);
    final List<MISTStream> op21List = Arrays.asList(op21);
    final List<MISTStream> op13List = Arrays.asList(op13);
    final List<MISTStream> op31List = Arrays.asList(op31);
    final List<MISTStream> op14List = Arrays.asList(op14);
    final List<MISTStream> sink1List = Arrays.asList(sink1);
    // Check src1 -> [op11->op12] edge
    final Map<List<MISTStream>, MISTEdge> e1 = operatorChainDag.getEdges(src1List);
    final Map<List<MISTStream>, MISTEdge> result1 = new HashMap<>();
    result1.put(op11op12, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check src2 -> [op21] edge
    final Map<List<MISTStream>, MISTEdge> e2 = operatorChainDag.getEdges(src2List);
    final Map<List<MISTStream>, MISTEdge> result2 = new HashMap<>();
    result2.put(op21List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e2, result2);
    // Check src3 -> [op31] edge
    final Map<List<MISTStream>, MISTEdge> e3 = operatorChainDag.getEdges(src3List);
    final Map<List<MISTStream>, MISTEdge> result3 = new HashMap<>();
    result3.put(op31List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e3, result3);
    // Check [op11->op12] -> [op13] edge
    final Map<List<MISTStream>, MISTEdge> e4 = operatorChainDag.getEdges(op11op12);
    final Map<List<MISTStream>, MISTEdge> result4 = new HashMap<>();
    result4.put(op13List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e4, result4);
    // Check [op21] -> [op13] edge
    final Map<List<MISTStream>, MISTEdge> e5 = operatorChainDag.getEdges(op21List);
    final Map<List<MISTStream>, MISTEdge> result5 = new HashMap<>();
    result5.put(op13List, new MISTEdge(Direction.RIGHT));
    Assert.assertEquals(e5, result5);
    // Check [op13] -> [op14] edge
    final Map<List<MISTStream>, MISTEdge> e6 = operatorChainDag.getEdges(op13List);
    final Map<List<MISTStream>, MISTEdge> result6 = new HashMap<>();
    result6.put(op14List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e6, result6);
    // Check [op31] -> [op14] edge
    final Map<List<MISTStream>, MISTEdge> e7 = operatorChainDag.getEdges(op31List);
    final Map<List<MISTStream>, MISTEdge> result7 = new HashMap<>();
    result7.put(op14List, new MISTEdge(Direction.RIGHT));
    Assert.assertEquals(e7, result7);
    // Check [op14] -> [sink1] edge
    final Map<List<MISTStream>, MISTEdge> e8 = operatorChainDag.getEdges(op14List);
    final Map<List<MISTStream>, MISTEdge> result8 = new HashMap<>();
    result8.put(sink1List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e8, result8);
}

Example 23 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamImpl method textSocketOutput.

@Override
public MISTStream<String> textSocketOutput(final String serverAddress, final int serverPort) {
    final Map<String, String> confMap = new HashMap<>();
    confMap.put(ConfKeys.SinkConf.SINK_TYPE.name(), ConfValues.SinkType.NETTY.name());
    confMap.put(ConfKeys.NettySink.SINK_ADDRESS.name(), serverAddress);
    confMap.put(ConfKeys.NettySink.SINK_PORT.name(), String.valueOf(serverPort));
    final MISTStream<String> sink = new MISTStreamImpl<>(dag, confMap);
    dag.addVertex(sink);
    dag.addEdge(this, sink, new MISTEdge(Direction.LEFT));
    return sink;
}

Example 24 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamImpl method transformToDoubleInputContinuousStream.

/**
 * Transform two upstreams to a new continuous stream
 * by applying the operation corresponding to the given configuration.
 * @param conf configuration
 * @param leftStream left stream
 * @param rightStream right stream
 * @param <OUT> output type
 * @return continuous stream
 */
private <OUT> ContinuousStream<OUT> transformToDoubleInputContinuousStream(final Map<String, String> conf, final MISTStream leftStream, final MISTStream rightStream) {
    final ContinuousStream<OUT> downStream = new ContinuousStreamImpl<>(dag, conf);
    dag.addVertex(downStream);
    dag.addEdge(leftStream, downStream, new MISTEdge(Direction.LEFT));
    dag.addEdge(rightStream, downStream, new MISTEdge(Direction.RIGHT));
    return downStream;
}

Example 25 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamImpl method transformToWindowedStream.

/**
 * Transform the upstream to a new windowed stream
 * by applying the operation corresponding to the given configuration.
 * @param conf configuration
 * @param upStream upstream
 * @param <OUT> output type
 * @return windowed stream
 */
private <OUT> WindowedStream<OUT> transformToWindowedStream(final Map<String, String> conf, final MISTStream upStream) {
    final WindowedStream<OUT> downStream = new WindowedStreamImpl<>(dag, conf);
    dag.addVertex(downStream);
    dag.addEdge(upStream, downStream, new MISTEdge(Direction.LEFT));
    return downStream;
}