Search in sources :

Example 21 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestQueryTranslator method testTranslateMultiSql.

@Test
public void testTranslateMultiSql() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
    String sql1 = "Insert into testavro.simpleOutputTopic select * from testavro.SIMPLE1";
    String sql2 = "Insert into testavro.SIMPLE3 select * from testavro.SIMPLE2";
    List<String> sqlStmts = Arrays.asList(sql1, sql2);
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(appDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), appDesc, 0);
    translator.translate(queryInfo.get(1), appDesc, 1);
    OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph();
    StreamConfig streamConfig = new StreamConfig(samzaConfig);
    String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get();
    String inputSystem1 = streamConfig.getSystem(inputStreamId1);
    String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1);
    String inputStreamId2 = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get();
    String inputSystem2 = streamConfig.getSystem(inputStreamId2);
    String inputPhysicalName2 = streamConfig.getPhysicalName(inputStreamId2);
    String outputStreamId1 = specGraph.getOutputStreams().keySet().stream().findFirst().get();
    String outputSystem1 = streamConfig.getSystem(outputStreamId1);
    String outputPhysicalName1 = streamConfig.getPhysicalName(outputStreamId1);
    String outputStreamId2 = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get();
    String outputSystem2 = streamConfig.getSystem(outputStreamId2);
    String outputPhysicalName2 = streamConfig.getPhysicalName(outputStreamId2);
    Assert.assertEquals(2, specGraph.getOutputStreams().size());
    Assert.assertEquals("testavro", outputSystem1);
    Assert.assertEquals("simpleOutputTopic", outputPhysicalName1);
    Assert.assertEquals("testavro", outputSystem2);
    Assert.assertEquals("SIMPLE3", outputPhysicalName2);
    Assert.assertEquals(2, specGraph.getInputOperators().size());
    Assert.assertEquals("testavro", inputSystem1);
    Assert.assertEquals("SIMPLE1", inputPhysicalName1);
    Assert.assertEquals("testavro", inputSystem2);
    Assert.assertEquals("SIMPLE2", inputPhysicalName2);
}
Also used : SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) OperatorSpecGraph(org.apache.samza.operators.OperatorSpecGraph) StreamConfig(org.apache.samza.config.StreamConfig) MapConfig(org.apache.samza.config.MapConfig) SamzaSqlTestConfig(org.apache.samza.sql.util.SamzaSqlTestConfig) SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) Config(org.apache.samza.config.Config) StreamConfig(org.apache.samza.config.StreamConfig) StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) Collection(java.util.Collection) MapConfig(org.apache.samza.config.MapConfig) SamzaSqlQueryParser(org.apache.samza.sql.util.SamzaSqlQueryParser) Test(org.junit.Test)

Example 22 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class KinesisSystemFactory method validateConfig.

protected void validateConfig(String system, Config config) {
    // Kinesis system does not support groupers other than AllSspToSingleTaskGrouper
    JobConfig jobConfig = new JobConfig(config);
    if (!jobConfig.getSystemStreamPartitionGrouperFactory().equals(AllSspToSingleTaskGrouperFactory.class.getCanonicalName())) {
        String errMsg = String.format("Incorrect Grouper %s used for KinesisSystemConsumer %s. Please set the %s config" + " to %s.", jobConfig.getSystemStreamPartitionGrouperFactory(), system, JobConfig.SSP_GROUPER_FACTORY, AllSspToSingleTaskGrouperFactory.class.getCanonicalName());
        throw new ConfigException(errMsg);
    }
    // Kinesis streams cannot be configured as broadcast streams
    TaskConfig taskConfig = new TaskConfig(config);
    if (taskConfig.getBroadcastSystemStreams().stream().anyMatch(ss -> system.equals(ss.getSystem()))) {
        throw new ConfigException("Kinesis streams cannot be configured as broadcast streams.");
    }
    // Kinesis streams cannot be configured as bootstrap streams
    KinesisConfig kConfig = new KinesisConfig(config);
    kConfig.getKinesisStreams(system).forEach(stream -> {
        StreamConfig streamConfig = new StreamConfig(kConfig);
        SystemStream ss = new SystemStream(system, stream);
        if (streamConfig.getBootstrapEnabled(ss)) {
            throw new ConfigException("Kinesis streams cannot be configured as bootstrap streams.");
        }
    });
}
Also used : AllSspToSingleTaskGrouperFactory(org.apache.samza.container.grouper.stream.AllSspToSingleTaskGrouperFactory) SystemStream(org.apache.samza.system.SystemStream) StreamConfig(org.apache.samza.config.StreamConfig) ConfigException(org.apache.samza.config.ConfigException) TaskConfig(org.apache.samza.config.TaskConfig) JobConfig(org.apache.samza.config.JobConfig)

Example 23 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestStreamUtil method testGetStreamWithOutSystemInConfig.

// System is required. Throw if it cannot be determined.
@Test(expected = IllegalArgumentException.class)
public void testGetStreamWithOutSystemInConfig() {
    Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME);
    StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
    assertEquals(TEST_SYSTEM, spec.getSystemName());
}
Also used : StreamSpec(org.apache.samza.system.StreamSpec) JobConfig(org.apache.samza.config.JobConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) StreamConfig(org.apache.samza.config.StreamConfig) StreamConfig(org.apache.samza.config.StreamConfig) Test(org.junit.Test)

Example 24 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestStreamUtil method testGetStreamWithSystemAtBothScopesInConfig.

// Stream scope should override default scope
@Test
public void testGetStreamWithSystemAtBothScopesInConfig() {
    Config config = addConfigs(buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, StreamConfig.SYSTEM, TEST_SYSTEM), JobConfig.JOB_DEFAULT_SYSTEM, TEST_DEFAULT_SYSTEM);
    StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
    assertEquals(TEST_SYSTEM, spec.getSystemName());
}
Also used : StreamSpec(org.apache.samza.system.StreamSpec) JobConfig(org.apache.samza.config.JobConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) StreamConfig(org.apache.samza.config.StreamConfig) StreamConfig(org.apache.samza.config.StreamConfig) Test(org.junit.Test)

Example 25 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestStreamUtil method testGetStreamPhysicalNameArgSpecialCharacters.

// Special characters are allowed for the physical name
@Test
public void testGetStreamPhysicalNameArgSpecialCharacters() {
    Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME_SPECIAL_CHARS, StreamConfig.SYSTEM, TEST_SYSTEM);
    StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
    assertEquals(TEST_PHYSICAL_NAME_SPECIAL_CHARS, spec.getPhysicalName());
}
Also used : StreamSpec(org.apache.samza.system.StreamSpec) JobConfig(org.apache.samza.config.JobConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) StreamConfig(org.apache.samza.config.StreamConfig) StreamConfig(org.apache.samza.config.StreamConfig) Test(org.junit.Test)

Aggregations

StreamConfig (org.apache.samza.config.StreamConfig)39 Config (org.apache.samza.config.Config)33 MapConfig (org.apache.samza.config.MapConfig)32 Test (org.junit.Test)31 JobConfig (org.apache.samza.config.JobConfig)23 StreamSpec (org.apache.samza.system.StreamSpec)17 Collection (java.util.Collection)11 StreamApplicationDescriptorImpl (org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl)10 OperatorSpecGraph (org.apache.samza.operators.OperatorSpecGraph)10 SamzaSqlApplicationConfig (org.apache.samza.sql.runner.SamzaSqlApplicationConfig)9 SamzaSqlQueryParser (org.apache.samza.sql.util.SamzaSqlQueryParser)9 SamzaSqlTestConfig (org.apache.samza.sql.util.SamzaSqlTestConfig)9 SystemStream (org.apache.samza.system.SystemStream)6 HashMap (java.util.HashMap)3 ApplicationConfig (org.apache.samza.config.ApplicationConfig)3 HashMultimap (com.google.common.collect.HashMultimap)2 Multimap (com.google.common.collect.Multimap)2 ArrayList (java.util.ArrayList)2 Collections (java.util.Collections)2 HashSet (java.util.HashSet)2