Examples with MISTQueryBuilder edu.snu.mist.client.MISTQueryBuilder used on opensource projects

Example 21 with MISTQueryBuilder

use of edu.snu.mist.client.MISTQueryBuilder in project mist by snuspl.

the class UnionMist method submitQuery.

/**
 * Submit a query unifying two sources.
 * The query reads strings from two source servers, unifies them, and send them to a sink server.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    final String source1Socket = injector.getNamedInstance(UnionLeftSourceAddress.class);
    final String source2Socket = injector.getNamedInstance(UnionRightSourceAddress.class);
    final SourceConfiguration localTextSocketSource1Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source1Socket);
    final SourceConfiguration localTextSocketSource2Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source2Socket);
    // Simple reduce function.
    final MISTBiFunction<Integer, Integer, Integer> reduceFunction = (v1, v2) -> {
        return v1 + v2;
    };
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream sourceStream1 = queryBuilder.socketTextStream(localTextSocketSource1Conf).map(s -> new Tuple2(s, 1));
    final ContinuousStream sourceStream2 = queryBuilder.socketTextStream(localTextSocketSource2Conf).map(s -> new Tuple2(s, 1));
    sourceStream1.union(sourceStream2).reduceByKey(0, String.class, reduceFunction).map(s -> s.toString()).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 22 with MISTQueryBuilder

use of edu.snu.mist.client.MISTQueryBuilder in project mist by snuspl.

the class WindowAndAggregate method submitQuery.

/**
 * Submit a windowing and aggregating query.
 * The query reads strings from a source server, puts them into window,
 * concatenates all inputs in the window into a single string using toString function, and
 * print out the start and end time of the window.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    // configurations for source and sink
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    // configurations for windowing and aggregation
    final int windowSize = 5000;
    final int windowEmissionInterval = 2500;
    final MISTFunction<WindowData<String>, String> aggregateFunc = (windowData) -> {
        return windowData.getDataCollection().toString() + ", window is started at " + windowData.getStart() + ", window is ended at " + windowData.getEnd() + ".";
    };
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).window(new TimeWindowInformation(windowSize, windowEmissionInterval)).aggregateWindow(aggregateFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 23 with MISTQueryBuilder

use of edu.snu.mist.client.MISTQueryBuilder in project mist by snuspl.

the class WordCount method submitQuery.

/**
 * Submit a query containing reduce-by-key operator.
 * The query reads strings from a source server, filters alphabetical words,
 * counts words using reduce-by-key operator, and sends them to a sink server.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    // Simple reduce function.
    final MISTBiFunction<Integer, Integer, Integer> reduceFunction = (v1, v2) -> {
        return v1 + v2;
    };
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).filter(s -> isAlpha(s)).map(s -> new Tuple2(s, 1)).reduceByKey(0, String.class, reduceFunction).map(s -> s.toString()).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 24 with MISTQueryBuilder

use of edu.snu.mist.client.MISTQueryBuilder in project mist by snuspl.

the class QueryInfoStoreTest method diskStoreTest.

/**
 * Tests whether the PlanStore correctly saves, deletes and loads the operator chain dag.
 * @throws InjectionException
 * @throws IOException
 */
@Test(timeout = 1000)
public void diskStoreTest() throws InjectionException, IOException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Jar files
    final List<ByteBuffer> jarFiles = new LinkedList<>();
    final ByteBuffer byteBuffer1 = ByteBuffer.wrap(new byte[] { 0, 1, 0, 1, 1, 1 });
    jarFiles.add(byteBuffer1);
    final Injector injector = Tang.Factory.getTang().newInjector();
    final QueryInfoStore store = injector.getInstance(QueryInfoStore.class);
    final ApplicationCodeManager applicationCodeManager = injector.getInstance(ApplicationCodeManager.class);
    final ApplicationMap applicationMap = injector.getInstance(ApplicationMap.class);
    final String queryId1 = "testQuery1";
    final String queryId2 = "testQuery2";
    final String tmpFolderPath = injector.getNamedInstance(SharedStorePath.class);
    final File folder = new File(tmpFolderPath);
    // Store jar files
    final List<String> paths = applicationCodeManager.registerNewAppCode(jarFiles).getJarPaths();
    for (int i = 0; i < jarFiles.size(); i++) {
        final ByteBuffer buf = ByteBuffer.allocateDirect(jarFiles.get(i).capacity());
        final String path = paths.get(i);
        final FileInputStream fis = new FileInputStream(path);
        final FileChannel channel = fis.getChannel();
        channel.read(buf);
        Assert.assertEquals(jarFiles.get(i), buf);
    }
    final ApplicationInfo applicationInfo = mock(ApplicationInfo.class);
    when(applicationInfo.getApplicationId()).thenReturn(TestParameters.SUPER_GROUP_ID);
    when(applicationInfo.getJarFilePath()).thenReturn(paths);
    applicationMap.putIfAbsent(TestParameters.SUPER_GROUP_ID, applicationInfo);
    // Generate logical plan
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroDag1 = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(paths).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final AvroDag avroDag2 = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(paths).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    // Store the chained dag
    store.saveAvroDag(new Tuple<>(queryId1, avroDag1));
    store.saveAvroDag(new Tuple<>(queryId2, avroDag2));
    while (!(store.isStored(queryId1) && store.isStored(queryId2))) {
    // Wait until the plan is stored
    }
    Assert.assertTrue(new File(tmpFolderPath, queryId1 + ".plan").exists());
    Assert.assertTrue(new File(tmpFolderPath, queryId2 + ".plan").exists());
    // Test stored file
    final AvroDag loadedDag1 = store.load(queryId1);
    Assert.assertEquals(avroDag1.getEdges(), loadedDag1.getEdges());
    Assert.assertEquals(avroDag1.getSchema(), loadedDag1.getSchema());
    testVerticesEqual(avroDag1.getAvroVertices(), loadedDag1.getAvroVertices());
    final AvroDag loadedDag2 = store.load(queryId2);
    Assert.assertEquals(avroDag2.getEdges(), loadedDag2.getEdges());
    Assert.assertEquals(avroDag2.getSchema(), loadedDag2.getSchema());
    testVerticesEqual(avroDag2.getAvroVertices(), loadedDag2.getAvroVertices());
    // Test deletion
    store.delete(queryId1);
    store.delete(queryId2);
    Assert.assertFalse(store.isStored(queryId1));
    Assert.assertFalse(new File(tmpFolderPath, queryId1 + ".plan").exists());
    Assert.assertFalse(store.isStored(queryId2));
    Assert.assertFalse(new File(tmpFolderPath, queryId2 + ".plan").exists());
    for (final String path : paths) {
        Assert.assertFalse(new File(path).exists());
    }
    folder.delete();
}

Example 25 with MISTQueryBuilder

use of edu.snu.mist.client.MISTQueryBuilder in project mist by snuspl.

the class HelloMist method submitQuery.

/**
 * Submit a stateless query.
 * The query reads strings from a source server, filter strings which start with "HelloMist:",
 * trim "HelloMist:" part of the filtered strings, and send them to a sink server.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).filter(s -> s.startsWith("HelloMIST:")).map(s -> s.substring("HelloMIST:".length()).trim()).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}