Examples with StreamApplicationDescriptorImpl org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl used on opensource projects

Example 41 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestQueryTranslator method testTranslateStreamStreamJoin.

@Test(expected = SamzaException.class)
public void testTranslateStreamStreamJoin() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1);
    String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW as pv" + " join testavro.PROFILE as p" + " on p.id = pv.profileId";
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<String> sqlStmts = fetchSqlFromConfig(config);
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), streamAppDesc, 0);
}

Example 42 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestQueryTranslator method testTranslateStreamTableJoinWithSubQuery.

@Test(expected = SamzaException.class)
public void testTranslateStreamTableJoinWithSubQuery() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1);
    String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW as pv" + " where exists " + " (select p.id from testavro.PROFILE.`$table` as p" + " where p.id = pv.profileId)";
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<String> sqlStmts = fetchSqlFromConfig(config);
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), streamAppDesc, 0);
}

Example 43 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestQueryTranslator method testTranslate.

@Test
public void testTranslate() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, "Insert into testavro.outputTopic(id) select MyTest(id) from testavro.level1.level2.SIMPLE1 as s where s.id = 10");
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<String> sqlStmts = fetchSqlFromConfig(config);
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(appDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), appDesc, 0);
    OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph();
    StreamConfig streamConfig = new StreamConfig(samzaConfig);
    String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get();
    String inputSystem = streamConfig.getSystem(inputStreamId);
    String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId);
    String outputStreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get();
    String outputSystem = streamConfig.getSystem(outputStreamId);
    String outputPhysicalName = streamConfig.getPhysicalName(outputStreamId);
    Assert.assertEquals(1, specGraph.getOutputStreams().size());
    Assert.assertEquals("testavro", outputSystem);
    Assert.assertEquals("outputTopic", outputPhysicalName);
    Assert.assertEquals(1, specGraph.getInputOperators().size());
    Assert.assertEquals("testavro", inputSystem);
    Assert.assertEquals("SIMPLE1", inputPhysicalName);
}

Example 44 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestQueryTranslator method testTranslateFanOut.

@Test
public void testTranslateFanOut() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
    String sql1 = "Insert into testavro.SIMPLE2 select * from testavro.SIMPLE1";
    String sql2 = "Insert into testavro.SIMPLE3 select * from testavro.SIMPLE1";
    List<String> sqlStmts = Arrays.asList(sql1, sql2);
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(appDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), appDesc, 0);
    translator.translate(queryInfo.get(1), appDesc, 1);
    OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph();
    StreamConfig streamConfig = new StreamConfig(samzaConfig);
    String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get();
    String inputSystem = streamConfig.getSystem(inputStreamId);
    String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId);
    String outputStreamId1 = specGraph.getOutputStreams().keySet().stream().findFirst().get();
    String outputSystem1 = streamConfig.getSystem(outputStreamId1);
    String outputPhysicalName1 = streamConfig.getPhysicalName(outputStreamId1);
    String outputStreamId2 = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get();
    String outputSystem2 = streamConfig.getSystem(outputStreamId2);
    String outputPhysicalName2 = streamConfig.getPhysicalName(outputStreamId2);
    Assert.assertEquals(2, specGraph.getOutputStreams().size());
    Assert.assertEquals("testavro", outputSystem1);
    Assert.assertEquals("SIMPLE2", outputPhysicalName1);
    Assert.assertEquals("testavro", outputSystem2);
    Assert.assertEquals("SIMPLE3", outputPhysicalName2);
    Assert.assertEquals(1, specGraph.getInputOperators().size());
    Assert.assertEquals("testavro", inputSystem);
    Assert.assertEquals("SIMPLE1", inputPhysicalName);
}

Example 45 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestQueryTranslator method testTranslateStreamTableJoinWithoutJoinOperator.

@Test(expected = SamzaException.class)
public void testTranslateStreamTableJoinWithoutJoinOperator() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1);
    String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW as pv, testavro.PROFILE.`$table` as p" + " where p.id = pv.profileId";
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<String> sqlStmts = fetchSqlFromConfig(config);
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), streamAppDesc, 0);
}