Examples with AvroDag edu.snu.mist.formats.avro.AvroDag used on opensource projects

Example 1 with AvroDag

use of edu.snu.mist.formats.avro.AvroDag in project mist by snuspl.

the class QueryIdGeneratorTest method testQueryIdGenerate.

/**
 * Test whether QueryIdGenerator generates query ids correctly.
 * It creates 10,000 query ids and checks the ids.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testQueryIdGenerate() throws InjectionException {
    final Injector injector = Tang.Factory.getTang().newInjector();
    final QueryIdGenerator queryIdGenerator = injector.getInstance(QueryIdGenerator.class);
    final String prefix = injector.getNamedInstance(QueryIdPrefix.class);
    final AvroDag avroDag = new AvroDag();
    long submittedQueryNum = 0;
    while (submittedQueryNum < 10000) {
        final String queryId = queryIdGenerator.generate();
        Assert.assertEquals(prefix + submittedQueryNum, queryId);
        submittedQueryNum++;
    }
}

Example 2 with AvroDag

use of edu.snu.mist.formats.avro.AvroDag in project mist by snuspl.

the class DefaultGroupCheckpointStore method loadSavedQueries.

@Override
public List<AvroDag> loadSavedQueries(final List<String> queryIdList) throws IOException {
    final List<AvroDag> savedQueries = new ArrayList<>();
    for (final String queryId : queryIdList) {
        final File storedFile = getQueryStoreFile(queryId);
        final DataFileReader<AvroDag> dataFileReader = new DataFileReader<>(storedFile, avroDagDatumReader);
        AvroDag avroDag = null;
        avroDag = dataFileReader.next(avroDag);
        savedQueries.add(avroDag);
    }
    return savedQueries;
}

Example 3 with AvroDag

use of edu.snu.mist.formats.avro.AvroDag in project mist by snuspl.

the class DefaultDagGeneratorImplTest method testPlanGenerator.

/**
 * Round-trip test of de-serializing AvroOperatorChainDag.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Generate avro operator chain dag
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
    final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
    final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
    final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
    final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
    // Test execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    final Set<ExecutionVertex> sources = dag.getRootVertices();
    Assert.assertEquals(1, sources.size());
    Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
    final PhysicalSource source = (PhysicalSource) sources.iterator().next();
    final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
    Assert.assertEquals(1, nextOps.size());
    final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
    final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
    final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
    final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
    final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
    Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
    Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
    Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
    Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
    Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}

Example 4 with AvroDag

use of edu.snu.mist.formats.avro.AvroDag in project mist by snuspl.

the class DefaultCheckpointManagerImpl method recoverGroup.

@Override
public void recoverGroup(final String groupId) throws IOException {
    final Map<String, QueryCheckpoint> queryCheckpointMap;
    final List<AvroDag> dagList;
    final QueryManager queryManager = queryManagerFuture.get();
    try {
        // Load the checkpointed states and the query lists.
        queryCheckpointMap = checkpointStore.loadSavedGroupState(groupId).getQueryCheckpointMap();
        final List<String> queryIdListInGroup = new ArrayList<>();
        for (final String queryId : queryCheckpointMap.keySet()) {
            queryIdListInGroup.add(queryId);
        }
        // Load the queries.
        dagList = checkpointStore.loadSavedQueries(queryIdListInGroup);
    } catch (final FileNotFoundException ie) {
        LOG.log(Level.WARNING, "Failed in loading group {0}, this group may not exist in the checkpoint store.", new Object[] { groupId });
        return;
    }
    for (final AvroDag avroDag : dagList) {
        final QueryCheckpoint queryCheckpoint = queryCheckpointMap.get(avroDag.getQueryId());
        // Recover each query in the group.
        queryManager.createQueryWithCheckpoint(avroDag, queryCheckpoint);
    }
}

Example 5 with AvroDag

use of edu.snu.mist.formats.avro.AvroDag in project mist by snuspl.

the class QueryManagerTest method testSubmitComplexQueryHelper.

@SuppressWarnings("unchecked")
private void testSubmitComplexQueryHelper(final Configuration conf) throws Exception {
    final String queryId = "testQuery";
    final List<String> inputs = Arrays.asList("mist is a cloud of tiny water droplets suspended in the atmosphere", "a mist rose out of the river", "the peaks were shrouded in mist");
    // Expected results
    final List<Map<String, Integer>> intermediateResult = getIntermediateResult(inputs);
    final List<String> expectedSink1Output = intermediateResult.stream().map(input -> input.toString()).collect(Collectors.toList());
    final List<Integer> expectedSink2Output = intermediateResult.stream().map(totalCountMapFunc).collect(Collectors.toList());
    // Number of expected outputs
    final CountDownLatch countDownAllOutputs = new CountDownLatch(intermediateResult.size() * 2);
    // Create the execution DAG of the query
    final ExecutionDag executionDag = new ExecutionDag(new AdjacentListDAG<>());
    // Create source
    final TestDataGenerator dataGenerator = new TestDataGenerator(inputs);
    final EventGenerator eventGenerator = new PunctuatedEventGenerator(null, input -> false, null, 0, null, null);
    final PhysicalSource src = new PhysicalSourceImpl("testSource", new HashMap<>(), dataGenerator, eventGenerator);
    final Injector injector = Tang.Factory.getTang().newInjector(conf);
    // Create sinks
    final List<String> sink1Result = new LinkedList<>();
    final List<Integer> sink2Result = new LinkedList<>();
    final PhysicalSink sink1 = new PhysicalSinkImpl<>("sink1", null, new TestWithCountDownSink<String>(sink1Result, countDownAllOutputs));
    final PhysicalSink sink2 = new PhysicalSinkImpl<>("sink2", null, new TestWithCountDownSink<Integer>(sink2Result, countDownAllOutputs));
    // Fake operator chain dag of QueryManager
    final AvroDag fakeAvroDag = new AvroDag();
    fakeAvroDag.setQueryId(queryId);
    // fakeAvroDag.setSuperGroupId("testGroup");
    // Construct execution dag
    constructExecutionDag(executionDag, src, sink1, sink2);
    // Create mock DagGenerator. It returns the above  execution dag
    final ConfigDagGenerator configDagGenerator = mock(ConfigDagGenerator.class);
    final DAG<ConfigVertex, MISTEdge> configDag = mock(DAG.class);
    when(configDagGenerator.generate(fakeAvroDag)).thenReturn(configDag);
    final DagGenerator dagGenerator = mock(DagGenerator.class);
    // when(dagGenerator.generate(configDag, tuple.getValue().getJarFilePaths())).thenReturn(executionDag);
    // Build QueryManager
    final QueryManager queryManager = queryManagerBuild(fakeAvroDag, configDagGenerator, dagGenerator, injector);
    queryManager.create(fakeAvroDag);
    // Wait until all of the outputs are generated
    countDownAllOutputs.await();
    // Check the outputs
    Assert.assertEquals(expectedSink1Output, sink1Result);
    Assert.assertEquals(expectedSink2Output, sink2Result);
    src.close();
    queryManager.close();
    // Delete plan directory and plans
    deletePlans(injector);
}