Examples with AvroVertex edu.snu.mist.formats.avro.AvroVertex used on opensource projects

Example 1 with AvroVertex

use of edu.snu.mist.formats.avro.AvroVertex in project mist by snuspl.

the class DefaultDagGeneratorImplTest method testPlanGenerator.

/**
 * Round-trip test of de-serializing AvroOperatorChainDag.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Generate avro operator chain dag
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
    final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
    final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
    final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
    final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
    // Test execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    final Set<ExecutionVertex> sources = dag.getRootVertices();
    Assert.assertEquals(1, sources.size());
    Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
    final PhysicalSource source = (PhysicalSource) sources.iterator().next();
    final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
    Assert.assertEquals(1, nextOps.size());
    final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
    final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
    final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
    final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
    final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
    Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
    Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
    Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
    Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
    Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}

Example 2 with AvroVertex

use of edu.snu.mist.formats.avro.AvroVertex in project mist by snuspl.

the class QueryInfoStoreTest method testVerticesEqual.

/**
 * Tests that two lists of vertices are equal.
 * @param vertices the first list of vertices
 * @param loadedVertices the second list of vertices
 */
private void testVerticesEqual(final List<AvroVertex> vertices, final List<AvroVertex> loadedVertices) {
    for (int i = 0; i < vertices.size(); i++) {
        final AvroVertex avroVertex = vertices.get(i);
        final AvroVertex loadedVertex = loadedVertices.get(i);
        Assert.assertEquals(avroVertex.getConfiguration(), loadedVertex.getConfiguration());
        Assert.assertEquals(avroVertex.getSchema(), loadedVertex.getSchema());
    }
}

Example 3 with AvroVertex

use of edu.snu.mist.formats.avro.AvroVertex in project mist by snuspl.

the class MISTQueryTest method mistComplexQuerySerializeTest.

/**
 * This method tests a serialization of a complex query, containing 9 vertices.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void mistComplexQuerySerializeTest() throws InjectionException, IOException, URISyntaxException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final ContinuousStream<String> sourceStream = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF, TestParameters.PUNCTUATED_WATERMARK_CONF);
    final ContinuousStream<String> flatMapStream = sourceStream.flatMap(expectedFlatMapFunc);
    final ContinuousStream<String> filterStream = flatMapStream.filter(expectedFilterPredicate);
    final ContinuousStream<Tuple2<String, Integer>> mapStream = filterStream.map(expectedMapFunc);
    final WindowedStream<Tuple2<String, Integer>> windowedStream = mapStream.window(new TimeWindowInformation(expectedWindowSize, expectedWindowEmissionInterval));
    final ContinuousStream<Map<String, Integer>> reduceByKeyStream = windowedStream.reduceByKeyWindow(0, String.class, expectedReduceFunc);
    final MISTStream<String> sinkStream = reduceByKeyStream.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    // Build a query
    final MISTQuery complexQuery = queryBuilder.build();
    final Tuple<List<AvroVertex>, List<Edge>> serializedDAG = complexQuery.getAvroOperatorDag();
    final List<AvroVertex> vertices = serializedDAG.getKey();
    Assert.assertEquals(7, vertices.size());
    Assert.assertEquals(sourceStream.getConfiguration(), vertices.get(0).getConfiguration());
    Assert.assertEquals(flatMapStream.getConfiguration(), vertices.get(1).getConfiguration());
    Assert.assertEquals(filterStream.getConfiguration(), vertices.get(2).getConfiguration());
    Assert.assertEquals(mapStream.getConfiguration(), vertices.get(3).getConfiguration());
    Assert.assertEquals(windowedStream.getConfiguration(), vertices.get(4).getConfiguration());
    Assert.assertEquals(reduceByKeyStream.getConfiguration(), vertices.get(5).getConfiguration());
    Assert.assertEquals(sinkStream.getConfiguration(), vertices.get(6).getConfiguration());
    final List<Edge> edges = serializedDAG.getValue();
    final List<Edge> expectedEdges = Arrays.asList(Edge.newBuilder().setFrom(0).setTo(1).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(1).setTo(2).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(2).setTo(3).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(3).setTo(4).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(4).setTo(5).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(5).setTo(6).setDirection(Direction.LEFT).setBranchIndex(0).build());
    Assert.assertEquals(new HashSet<>(expectedEdges), new HashSet<>(edges));
}

Example 4 with AvroVertex

use of edu.snu.mist.formats.avro.AvroVertex in project mist by snuspl.

the class MISTQueryImpl method getAvroOperatorDag.

@Override
public Tuple<List<AvroVertex>, List<Edge>> getAvroOperatorDag() {
    final LogicalDagOptimizer logicalDagOptimizer = new LogicalDagOptimizer(dag);
    final DAG<MISTStream, MISTEdge> optimizedDag = logicalDagOptimizer.getOptimizedDAG();
    final Queue<MISTStream> queue = new LinkedList<>();
    final List<MISTStream> vertices = new ArrayList<>();
    final List<Edge> edges = new ArrayList<>();
    // Put all vertices into a queue
    final Iterator<MISTStream> iterator = GraphUtils.topologicalSort(optimizedDag);
    while (iterator.hasNext()) {
        final MISTStream vertex = iterator.next();
        queue.add(vertex);
        vertices.add(vertex);
    }
    // Visit each vertex and serialize its edges
    while (!queue.isEmpty()) {
        final MISTStream vertex = queue.remove();
        final int fromIndex = vertices.indexOf(vertex);
        final Map<MISTStream, MISTEdge> neighbors = optimizedDag.getEdges(vertex);
        for (final Map.Entry<MISTStream, MISTEdge> neighbor : neighbors.entrySet()) {
            final int toIndex = vertices.indexOf(neighbor.getKey());
            final MISTEdge edgeInfo = neighbor.getValue();
            final Edge.Builder edgeBuilder = Edge.newBuilder().setFrom(fromIndex).setTo(toIndex).setDirection(edgeInfo.getDirection()).setBranchIndex(edgeInfo.getIndex());
            edges.add(edgeBuilder.build());
        }
    }
    final Set<MISTStream> rootVertices = optimizedDag.getRootVertices();
    // Serialize each vertex via avro.
    final List<AvroVertex> serializedVertices = new ArrayList<>();
    for (final MISTStream vertex : vertices) {
        final AvroVertex.Builder vertexBuilder = AvroVertex.newBuilder();
        vertexBuilder.setConfiguration(vertex.getConfiguration());
        vertexBuilder.setVertexId(String.valueOf(vertexIdIndex));
        // Set vertex type
        if (rootVertices.contains(vertex)) {
            // this is a source
            vertexBuilder.setAvroVertexType(AvroVertexTypeEnum.SOURCE);
        } else if (optimizedDag.getEdges(vertex).size() == 0) {
            // this is a sink
            vertexBuilder.setAvroVertexType(AvroVertexTypeEnum.SINK);
        } else {
            vertexBuilder.setAvroVertexType(AvroVertexTypeEnum.OPERATOR);
        }
        serializedVertices.add(vertexBuilder.build());
        vertexIdIndex++;
    }
    return new Tuple<>(serializedVertices, edges);
}

Example 5 with AvroVertex

use of edu.snu.mist.formats.avro.AvroVertex in project mist by snuspl.

the class QueryInfoStoreTest method diskStoreTest.

/**
 * Tests whether the PlanStore correctly saves, deletes and loads the operator chain dag.
 * @throws InjectionException
 * @throws IOException
 */
@Test(timeout = 1000)
public void diskStoreTest() throws InjectionException, IOException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Jar files
    final List<ByteBuffer> jarFiles = new LinkedList<>();
    final ByteBuffer byteBuffer1 = ByteBuffer.wrap(new byte[] { 0, 1, 0, 1, 1, 1 });
    jarFiles.add(byteBuffer1);
    final Injector injector = Tang.Factory.getTang().newInjector();
    final QueryInfoStore store = injector.getInstance(QueryInfoStore.class);
    final ApplicationCodeManager applicationCodeManager = injector.getInstance(ApplicationCodeManager.class);
    final ApplicationMap applicationMap = injector.getInstance(ApplicationMap.class);
    final String queryId1 = "testQuery1";
    final String queryId2 = "testQuery2";
    final String tmpFolderPath = injector.getNamedInstance(SharedStorePath.class);
    final File folder = new File(tmpFolderPath);
    // Store jar files
    final List<String> paths = applicationCodeManager.registerNewAppCode(jarFiles).getJarPaths();
    for (int i = 0; i < jarFiles.size(); i++) {
        final ByteBuffer buf = ByteBuffer.allocateDirect(jarFiles.get(i).capacity());
        final String path = paths.get(i);
        final FileInputStream fis = new FileInputStream(path);
        final FileChannel channel = fis.getChannel();
        channel.read(buf);
        Assert.assertEquals(jarFiles.get(i), buf);
    }
    final ApplicationInfo applicationInfo = mock(ApplicationInfo.class);
    when(applicationInfo.getApplicationId()).thenReturn(TestParameters.SUPER_GROUP_ID);
    when(applicationInfo.getJarFilePath()).thenReturn(paths);
    applicationMap.putIfAbsent(TestParameters.SUPER_GROUP_ID, applicationInfo);
    // Generate logical plan
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroDag1 = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(paths).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final AvroDag avroDag2 = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(paths).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    // Store the chained dag
    store.saveAvroDag(new Tuple<>(queryId1, avroDag1));
    store.saveAvroDag(new Tuple<>(queryId2, avroDag2));
    while (!(store.isStored(queryId1) && store.isStored(queryId2))) {
    // Wait until the plan is stored
    }
    Assert.assertTrue(new File(tmpFolderPath, queryId1 + ".plan").exists());
    Assert.assertTrue(new File(tmpFolderPath, queryId2 + ".plan").exists());
    // Test stored file
    final AvroDag loadedDag1 = store.load(queryId1);
    Assert.assertEquals(avroDag1.getEdges(), loadedDag1.getEdges());
    Assert.assertEquals(avroDag1.getSchema(), loadedDag1.getSchema());
    testVerticesEqual(avroDag1.getAvroVertices(), loadedDag1.getAvroVertices());
    final AvroDag loadedDag2 = store.load(queryId2);
    Assert.assertEquals(avroDag2.getEdges(), loadedDag2.getEdges());
    Assert.assertEquals(avroDag2.getSchema(), loadedDag2.getSchema());
    testVerticesEqual(avroDag2.getAvroVertices(), loadedDag2.getAvroVertices());
    // Test deletion
    store.delete(queryId1);
    store.delete(queryId2);
    Assert.assertFalse(store.isStored(queryId1));
    Assert.assertFalse(new File(tmpFolderPath, queryId1 + ".plan").exists());
    Assert.assertFalse(store.isStored(queryId2));
    Assert.assertFalse(new File(tmpFolderPath, queryId2 + ".plan").exists());
    for (final String path : paths) {
        Assert.assertFalse(new File(path).exists());
    }
    folder.delete();
}