use of org.apache.samza.config.StreamConfig in project samza by apache.
the class TestQueryTranslator method testTranslateMultiSql.
@Test
public void testTranslateMultiSql() {
Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
String sql1 = "Insert into testavro.simpleOutputTopic select * from testavro.SIMPLE1";
String sql2 = "Insert into testavro.SIMPLE3 select * from testavro.SIMPLE2";
List<String> sqlStmts = Arrays.asList(sql1, sql2);
config.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(streamApp -> {
}, samzaConfig);
QueryTranslator translator = new QueryTranslator(appDesc, samzaSqlApplicationConfig);
translator.translate(queryInfo.get(0), appDesc, 0);
translator.translate(queryInfo.get(1), appDesc, 1);
OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph();
StreamConfig streamConfig = new StreamConfig(samzaConfig);
String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get();
String inputSystem1 = streamConfig.getSystem(inputStreamId1);
String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1);
String inputStreamId2 = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get();
String inputSystem2 = streamConfig.getSystem(inputStreamId2);
String inputPhysicalName2 = streamConfig.getPhysicalName(inputStreamId2);
String outputStreamId1 = specGraph.getOutputStreams().keySet().stream().findFirst().get();
String outputSystem1 = streamConfig.getSystem(outputStreamId1);
String outputPhysicalName1 = streamConfig.getPhysicalName(outputStreamId1);
String outputStreamId2 = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get();
String outputSystem2 = streamConfig.getSystem(outputStreamId2);
String outputPhysicalName2 = streamConfig.getPhysicalName(outputStreamId2);
Assert.assertEquals(2, specGraph.getOutputStreams().size());
Assert.assertEquals("testavro", outputSystem1);
Assert.assertEquals("simpleOutputTopic", outputPhysicalName1);
Assert.assertEquals("testavro", outputSystem2);
Assert.assertEquals("SIMPLE3", outputPhysicalName2);
Assert.assertEquals(2, specGraph.getInputOperators().size());
Assert.assertEquals("testavro", inputSystem1);
Assert.assertEquals("SIMPLE1", inputPhysicalName1);
Assert.assertEquals("testavro", inputSystem2);
Assert.assertEquals("SIMPLE2", inputPhysicalName2);
}
use of org.apache.samza.config.StreamConfig in project samza by apache.
the class KinesisSystemFactory method validateConfig.
protected void validateConfig(String system, Config config) {
// Kinesis system does not support groupers other than AllSspToSingleTaskGrouper
JobConfig jobConfig = new JobConfig(config);
if (!jobConfig.getSystemStreamPartitionGrouperFactory().equals(AllSspToSingleTaskGrouperFactory.class.getCanonicalName())) {
String errMsg = String.format("Incorrect Grouper %s used for KinesisSystemConsumer %s. Please set the %s config" + " to %s.", jobConfig.getSystemStreamPartitionGrouperFactory(), system, JobConfig.SSP_GROUPER_FACTORY, AllSspToSingleTaskGrouperFactory.class.getCanonicalName());
throw new ConfigException(errMsg);
}
// Kinesis streams cannot be configured as broadcast streams
TaskConfig taskConfig = new TaskConfig(config);
if (taskConfig.getBroadcastSystemStreams().stream().anyMatch(ss -> system.equals(ss.getSystem()))) {
throw new ConfigException("Kinesis streams cannot be configured as broadcast streams.");
}
// Kinesis streams cannot be configured as bootstrap streams
KinesisConfig kConfig = new KinesisConfig(config);
kConfig.getKinesisStreams(system).forEach(stream -> {
StreamConfig streamConfig = new StreamConfig(kConfig);
SystemStream ss = new SystemStream(system, stream);
if (streamConfig.getBootstrapEnabled(ss)) {
throw new ConfigException("Kinesis streams cannot be configured as bootstrap streams.");
}
});
}
use of org.apache.samza.config.StreamConfig in project samza by apache.
the class TestStreamUtil method testGetStreamWithOutSystemInConfig.
// System is required. Throw if it cannot be determined.
@Test(expected = IllegalArgumentException.class)
public void testGetStreamWithOutSystemInConfig() {
Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME);
StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
assertEquals(TEST_SYSTEM, spec.getSystemName());
}
use of org.apache.samza.config.StreamConfig in project samza by apache.
the class TestStreamUtil method testGetStreamWithSystemAtBothScopesInConfig.
// Stream scope should override default scope
@Test
public void testGetStreamWithSystemAtBothScopesInConfig() {
Config config = addConfigs(buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, StreamConfig.SYSTEM, TEST_SYSTEM), JobConfig.JOB_DEFAULT_SYSTEM, TEST_DEFAULT_SYSTEM);
StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
assertEquals(TEST_SYSTEM, spec.getSystemName());
}
use of org.apache.samza.config.StreamConfig in project samza by apache.
the class TestStreamUtil method testGetStreamPhysicalNameArgSpecialCharacters.
// Special characters are allowed for the physical name
@Test
public void testGetStreamPhysicalNameArgSpecialCharacters() {
Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME_SPECIAL_CHARS, StreamConfig.SYSTEM, TEST_SYSTEM);
StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
assertEquals(TEST_PHYSICAL_NAME_SPECIAL_CHARS, spec.getPhysicalName());
}
Aggregations