Examples with MISTEdge edu.snu.mist.common.graph.MISTEdge used on opensource projects

Example 41 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamTest method testTimeWindowedStream.

/**
 * Test for creating time-based WindowedStream from ContinuousStream.
 */
@Test
public void testTimeWindowedStream() {
    final WindowedStream<Tuple2<String, Integer>> timeWindowedStream = filteredMappedStream.window(new TimeWindowInformation(windowSize, windowEmissionInterval));
    final Map<String, String> conf = timeWindowedStream.getConfiguration();
    checkSizeBasedWindowInfo(windowSize, windowEmissionInterval, conf);
    // Check map -> timeWindow
    checkEdges(queryBuilder.build().getDAG(), 1, filteredMappedStream, timeWindowedStream, new MISTEdge(Direction.LEFT));
}

Example 42 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamTest method testMqttSink.

/**
 * Test for Mqtt sink.
 */
@Test
public void testMqttSink() throws InjectionException {
    final MISTStream<MqttMessage> sink = filteredMappedStream.mqttOutput(TestParameters.HOST, TestParameters.TOPIC);
    final Map<String, String> conf = sink.getConfiguration();
    Assert.assertEquals(TestParameters.HOST, conf.get(ConfKeys.MqttSink.MQTT_SINK_BROKER_URI.name()));
    Assert.assertEquals(TestParameters.TOPIC, conf.get(ConfKeys.MqttSink.MQTT_SINK_TOPIC.name()));
    // Check src -> sink
    final DAG<MISTStream, MISTEdge> dag = queryBuilder.build().getDAG();
    final Map<MISTStream, MISTEdge> neighbors = dag.getEdges(filteredMappedStream);
    Assert.assertEquals(1, neighbors.size());
    Assert.assertEquals(new MISTEdge(Direction.LEFT), neighbors.get(sink));
}

Example 43 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamTest method testReduceByKeyOperatorStream.

/**
 * Test for reduceByKey operator.
 */
@Test
public void testReduceByKeyOperatorStream() throws IOException {
    final MISTBiFunction<Integer, Integer, Integer> biFunc = (x, y) -> x + y;
    final int keyIndex = 0;
    final ContinuousStream<Map<String, Integer>> reducedStream = filteredMappedStream.reduceByKey(0, String.class, biFunc);
    final Map<String, String> conf = reducedStream.getConfiguration();
    Assert.assertEquals(String.valueOf(keyIndex), conf.get(ConfKeys.ReduceByKeyOperator.KEY_INDEX.name()));
    Assert.assertEquals(SerializeUtils.serializeToString(biFunc), conf.get(ConfKeys.ReduceByKeyOperator.MIST_BI_FUNC.name()));
    // Check filter -> map -> reduceBy
    checkEdges(queryBuilder.build().getDAG(), 1, filteredMappedStream, reducedStream, new MISTEdge(Direction.LEFT));
}

Example 44 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class OperatorChainDagGenerator method generateOperatorChainDAG.

/**
 * Generate OperatorChain DAG according to the logic described above.
 * @return the OperatorChain DAG
 * The chain is represented as a list and AvroVertexSerializable can be serialized by avro
 */
public DAG<List<MISTStream>, MISTEdge> generateOperatorChainDAG() {
    final DAG<OperatorChain, MISTEdge> chainDag = new AdjacentListDAG<>();
    final Map<MISTStream, OperatorChain> vertexChainMap = new HashMap<>();
    // from the root operators which are following sources.
    for (final MISTStream source : optimizedDag.getRootVertices()) {
        final Map<MISTStream, MISTEdge> rootEdges = optimizedDag.getEdges(source);
        // This chaining group is a wrapper for List, for equality check
        final OperatorChain srcChain = new OperatorChain();
        // Partition Source
        srcChain.chain.add(source);
        chainDag.addVertex(srcChain);
        vertexChainMap.put(source, srcChain);
        for (final Map.Entry<MISTStream, MISTEdge> entry : rootEdges.entrySet()) {
            final MISTStream nextVertex = entry.getKey();
            final MISTEdge edge = entry.getValue();
            final OperatorChain nextChain = getOperatorChain(nextVertex, vertexChainMap, chainDag);
            chainDag.addEdge(srcChain, nextChain, edge);
            chainingInDfsOrder(nextChain, nextVertex, chainDag, vertexChainMap);
        }
    }
    // Convert to List<AvroVertexSerializable> for AvroOperatorChainDag
    final DAG<List<MISTStream>, MISTEdge> result = new AdjacentListDAG<>();
    final Queue<OperatorChain> queue = new LinkedList<>();
    final Iterator<OperatorChain> iterator = GraphUtils.topologicalSort(chainDag);
    while (iterator.hasNext()) {
        final OperatorChain queryPartition = iterator.next();
        queue.add(queryPartition);
        result.addVertex(queryPartition.chain);
    }
    for (final OperatorChain operatorChain : queue) {
        final Map<OperatorChain, MISTEdge> edges = chainDag.getEdges(operatorChain);
        for (final Map.Entry<OperatorChain, MISTEdge> edge : edges.entrySet()) {
            result.addEdge(operatorChain.chain, edge.getKey().chain, edge.getValue());
        }
    }
    return result;
}

Example 45 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamImpl method mqttOutput.

@Override
public MISTStream<MqttMessage> mqttOutput(final String brokerURI, final String topic) {
    final Map<String, String> confMap = new HashMap<>();
    confMap.put(ConfKeys.SinkConf.SINK_TYPE.name(), ConfValues.SinkType.MQTT.name());
    confMap.put(ConfKeys.MqttSink.MQTT_SINK_BROKER_URI.name(), brokerURI);
    confMap.put(ConfKeys.MqttSink.MQTT_SINK_TOPIC.name(), topic);
    final MISTStream<MqttMessage> sink = new MISTStreamImpl<>(dag, confMap);
    dag.addVertex(sink);
    dag.addEdge(this, sink, new MISTEdge(Direction.LEFT));
    return sink;
}