Examples with AvroDag edu.snu.mist.formats.avro.AvroDag used on opensource projects

Example 6 with AvroDag

use of edu.snu.mist.formats.avro.AvroDag in project mist by snuspl.

the class GroupRecoveryTest method testSingleQueryRecovery.

@Test(timeout = 500000)
public void testSingleQueryRecovery() throws Exception {
    // Start source servers.
    final CountDownLatch sourceCountDownLatch1 = new CountDownLatch(1);
    final NettyTextMessageStreamGenerator textMessageStreamGenerator1 = new NettyTextMessageStreamGenerator(SERVER_ADDR, SOURCE_PORT1, new TestChannelHandler(sourceCountDownLatch1));
    final TestSinkHandler handler1 = new TestSinkHandler();
    final NettyTextMessageOutputReceiver receiver1 = new NettyTextMessageOutputReceiver("localhost", SINK_PORT, handler1);
    // Submit query.
    final MISTQuery query = buildQuery();
    // Generate avro chained dag : needed parts from MISTExamplesUtils.submitQuery()
    final Tuple<List<AvroVertex>, List<Edge>> initialAvroOpChainDag = query.getAvroOperatorDag();
    final String appId = "testApp";
    final String queryId = "testQuery";
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroDag = avroDagBuilder.setAppId(appId).setQueryId(queryId).setJarPaths(new ArrayList<>()).setAvroVertices(initialAvroOpChainDag.getKey()).setEdges(initialAvroOpChainDag.getValue()).build();
    // Build QueryManager.
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(PeriodicCheckpointPeriod.class, "1000");
    jcb.bindImplementation(QueryManager.class, GroupAwareQueryManagerImpl.class);
    jcb.bindImplementation(QueryStarter.class, ImmediateQueryMergingStarter.class);
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    injector.bindVolatileInstance(GroupIdRequestor.class, new TestGroupIdRequestor());
    injector.bindVolatileInstance(TaskStatsUpdater.class, mock(TaskStatsUpdater.class));
    final CheckpointManager checkpointManager = injector.getInstance(CheckpointManager.class);
    final QueryManager queryManager = injector.getInstance(QueryManager.class);
    queryManager.createApplication(appId, Arrays.asList(""));
    queryManager.create(avroDag);
    // Wait until all sources connect to stream generator
    sourceCountDownLatch1.await();
    final ExecutionDags executionDags = checkpointManager.getApplication(appId).getGroups().get(0).getExecutionDags();
    Assert.assertEquals(executionDags.values().size(), 1);
    final ExecutionDag executionDag = executionDags.values().iterator().next();
    // 1st stage. Push inputs to all sources and see if the results are proper.
    SRC0INPUT1.forEach(textMessageStreamGenerator1::write);
    LATCH1.await();
    Assert.assertEquals("{aa=2, bb=1, cc=1}", handler1.getResults().get(handler1.getResults().size() - 1));
    // Sleep 2 seconds for the checkpoint events to be sent out.
    sleep(2000);
    // Checkpoint the entire MISTTask, delete it, and restore it to see if it works.
    final String groupId = checkpointManager.getApplication(appId).getGroups().get(0).getGroupId();
    checkpointManager.checkpointGroup(groupId);
    checkpointManager.deleteGroup(groupId);
    // Close the generator.
    textMessageStreamGenerator1.close();
    receiver1.close();
    // Restart source servers.
    final CountDownLatch sourceCountDownLatch2 = new CountDownLatch(1);
    final NettyTextMessageStreamGenerator textMessageStreamGenerator2 = new NettyTextMessageStreamGenerator(SERVER_ADDR, SOURCE_PORT1, new TestChannelHandler(sourceCountDownLatch2));
    final TestSinkHandler handler2 = new TestSinkHandler();
    final NettyTextMessageOutputReceiver receiver2 = new NettyTextMessageOutputReceiver("localhost", SINK_PORT, handler2);
    // Recover the group.
    checkpointManager.recoverGroup(groupId);
    // Wait until all sources connect to stream generator
    sourceCountDownLatch2.await();
    final ExecutionDags executionDags2 = checkpointManager.getApplication(appId).getGroups().get(0).getExecutionDags();
    Assert.assertEquals(executionDags2.values().size(), 1);
    final ExecutionDag executionDag2 = executionDags2.values().iterator().next();
    // 2nd stage. Push inputs to the recovered the query to see if it works.
    SRC0INPUT2.forEach(textMessageStreamGenerator2::write);
    LATCH2.await();
    Assert.assertEquals("{aa=3, bb=3, cc=3}", handler2.getResults().get(handler2.getResults().size() - 1));
    // Close the generators.
    textMessageStreamGenerator2.close();
    receiver2.close();
}

Example 7 with AvroDag

use of edu.snu.mist.formats.avro.AvroDag in project mist by snuspl.

the class DefaultGroupCheckpointStore method saveQuery.

@Override
public boolean saveQuery(final AvroDag avroDag) {
    final String queryId = avroDag.getQueryId();
    try {
        final File storedFile = getQueryStoreFile(queryId);
        if (storedFile.exists()) {
            storedFile.delete();
            LOG.log(Level.INFO, "Deleting a duplicate query file");
        }
        final DataFileWriter<AvroDag> dataFileWriter = new DataFileWriter<>(avroDagDatumWriter);
        dataFileWriter.create(avroDag.getSchema(), storedFile);
        dataFileWriter.append(avroDag);
        dataFileWriter.close();
        LOG.log(Level.INFO, "Query {0} has been stored to disk.", queryId);
        return true;
    } catch (final Exception e) {
        e.printStackTrace();
        return false;
    }
}

Example 8 with AvroDag

use of edu.snu.mist.formats.avro.AvroDag in project mist by snuspl.

the class AsyncDiskQueryInfoStore method loadFromFile.

/**
 * Load the stored dag from File.
 * @param storedPlanFile file
 * @return chained dag
 * @throws IOException
 */
private AvroDag loadFromFile(final File storedPlanFile) throws IOException {
    final DataFileReader<AvroDag> dataFileReader = new DataFileReader<AvroDag>(storedPlanFile, datumReader);
    AvroDag dag = null;
    dag = dataFileReader.next(dag);
    return dag;
}

Example 9 with AvroDag

use of edu.snu.mist.formats.avro.AvroDag in project mist by snuspl.

the class AsyncDiskQueryInfoStore method delete.

/**
 * Deletes the dag and jar files.
 * @param queryId
 * @throws IOException
 */
@Override
public void delete(final String queryId) throws IOException {
    storedQuerySet.remove(queryId);
    final File storedFile = getAvroOperatorChainDagFile(queryId);
    final AvroDag logicalPlan = loadFromFile(storedFile);
    final String appId = logicalPlan.getAppId();
    final List<String> paths = metaApplicationMap.get(appId).getJarFilePath();
    // Delete jar files for the query
    for (final String path : paths) {
        if (path.startsWith(tmpFolderPath)) {
            // Delete the jar file if it is in the temp folder
            final File jarFile = new File(path);
            if (jarFile.exists()) {
                jarFile.delete();
            }
        }
    }
    // Delete the logical plan
    if (storedFile.exists()) {
        storedFile.delete();
    }
}

Example 10 with AvroDag

use of edu.snu.mist.formats.avro.AvroDag in project mist by snuspl.

the class QueryInfoStoreTest method diskStoreTest.

/**
 * Tests whether the PlanStore correctly saves, deletes and loads the operator chain dag.
 * @throws InjectionException
 * @throws IOException
 */
@Test(timeout = 1000)
public void diskStoreTest() throws InjectionException, IOException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Jar files
    final List<ByteBuffer> jarFiles = new LinkedList<>();
    final ByteBuffer byteBuffer1 = ByteBuffer.wrap(new byte[] { 0, 1, 0, 1, 1, 1 });
    jarFiles.add(byteBuffer1);
    final Injector injector = Tang.Factory.getTang().newInjector();
    final QueryInfoStore store = injector.getInstance(QueryInfoStore.class);
    final ApplicationCodeManager applicationCodeManager = injector.getInstance(ApplicationCodeManager.class);
    final ApplicationMap applicationMap = injector.getInstance(ApplicationMap.class);
    final String queryId1 = "testQuery1";
    final String queryId2 = "testQuery2";
    final String tmpFolderPath = injector.getNamedInstance(SharedStorePath.class);
    final File folder = new File(tmpFolderPath);
    // Store jar files
    final List<String> paths = applicationCodeManager.registerNewAppCode(jarFiles).getJarPaths();
    for (int i = 0; i < jarFiles.size(); i++) {
        final ByteBuffer buf = ByteBuffer.allocateDirect(jarFiles.get(i).capacity());
        final String path = paths.get(i);
        final FileInputStream fis = new FileInputStream(path);
        final FileChannel channel = fis.getChannel();
        channel.read(buf);
        Assert.assertEquals(jarFiles.get(i), buf);
    }
    final ApplicationInfo applicationInfo = mock(ApplicationInfo.class);
    when(applicationInfo.getApplicationId()).thenReturn(TestParameters.SUPER_GROUP_ID);
    when(applicationInfo.getJarFilePath()).thenReturn(paths);
    applicationMap.putIfAbsent(TestParameters.SUPER_GROUP_ID, applicationInfo);
    // Generate logical plan
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroDag1 = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(paths).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final AvroDag avroDag2 = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(paths).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    // Store the chained dag
    store.saveAvroDag(new Tuple<>(queryId1, avroDag1));
    store.saveAvroDag(new Tuple<>(queryId2, avroDag2));
    while (!(store.isStored(queryId1) && store.isStored(queryId2))) {
    // Wait until the plan is stored
    }
    Assert.assertTrue(new File(tmpFolderPath, queryId1 + ".plan").exists());
    Assert.assertTrue(new File(tmpFolderPath, queryId2 + ".plan").exists());
    // Test stored file
    final AvroDag loadedDag1 = store.load(queryId1);
    Assert.assertEquals(avroDag1.getEdges(), loadedDag1.getEdges());
    Assert.assertEquals(avroDag1.getSchema(), loadedDag1.getSchema());
    testVerticesEqual(avroDag1.getAvroVertices(), loadedDag1.getAvroVertices());
    final AvroDag loadedDag2 = store.load(queryId2);
    Assert.assertEquals(avroDag2.getEdges(), loadedDag2.getEdges());
    Assert.assertEquals(avroDag2.getSchema(), loadedDag2.getSchema());
    testVerticesEqual(avroDag2.getAvroVertices(), loadedDag2.getAvroVertices());
    // Test deletion
    store.delete(queryId1);
    store.delete(queryId2);
    Assert.assertFalse(store.isStored(queryId1));
    Assert.assertFalse(new File(tmpFolderPath, queryId1 + ".plan").exists());
    Assert.assertFalse(store.isStored(queryId2));
    Assert.assertFalse(new File(tmpFolderPath, queryId2 + ".plan").exists());
    for (final String path : paths) {
        Assert.assertFalse(new File(path).exists());
    }
    folder.delete();
}