Examples with StreamConfig org.apache.samza.config.StreamConfig used on opensource projects

Example 21 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestQueryTranslator method testTranslateMultiSql.

@Test
public void testTranslateMultiSql() {
    Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
    String sql1 = "Insert into testavro.simpleOutputTopic select * from testavro.SIMPLE1";
    String sql2 = "Insert into testavro.SIMPLE3 select * from testavro.SIMPLE2";
    List<String> sqlStmts = Arrays.asList(sql1, sql2);
    config.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
    Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
    List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
    SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream).collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
    StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(streamApp -> {
    }, samzaConfig);
    QueryTranslator translator = new QueryTranslator(appDesc, samzaSqlApplicationConfig);
    translator.translate(queryInfo.get(0), appDesc, 0);
    translator.translate(queryInfo.get(1), appDesc, 1);
    OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph();
    StreamConfig streamConfig = new StreamConfig(samzaConfig);
    String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get();
    String inputSystem1 = streamConfig.getSystem(inputStreamId1);
    String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1);
    String inputStreamId2 = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get();
    String inputSystem2 = streamConfig.getSystem(inputStreamId2);
    String inputPhysicalName2 = streamConfig.getPhysicalName(inputStreamId2);
    String outputStreamId1 = specGraph.getOutputStreams().keySet().stream().findFirst().get();
    String outputSystem1 = streamConfig.getSystem(outputStreamId1);
    String outputPhysicalName1 = streamConfig.getPhysicalName(outputStreamId1);
    String outputStreamId2 = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get();
    String outputSystem2 = streamConfig.getSystem(outputStreamId2);
    String outputPhysicalName2 = streamConfig.getPhysicalName(outputStreamId2);
    Assert.assertEquals(2, specGraph.getOutputStreams().size());
    Assert.assertEquals("testavro", outputSystem1);
    Assert.assertEquals("simpleOutputTopic", outputPhysicalName1);
    Assert.assertEquals("testavro", outputSystem2);
    Assert.assertEquals("SIMPLE3", outputPhysicalName2);
    Assert.assertEquals(2, specGraph.getInputOperators().size());
    Assert.assertEquals("testavro", inputSystem1);
    Assert.assertEquals("SIMPLE1", inputPhysicalName1);
    Assert.assertEquals("testavro", inputSystem2);
    Assert.assertEquals("SIMPLE2", inputPhysicalName2);
}

Example 22 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class KinesisSystemFactory method validateConfig.

protected void validateConfig(String system, Config config) {
    // Kinesis system does not support groupers other than AllSspToSingleTaskGrouper
    JobConfig jobConfig = new JobConfig(config);
    if (!jobConfig.getSystemStreamPartitionGrouperFactory().equals(AllSspToSingleTaskGrouperFactory.class.getCanonicalName())) {
        String errMsg = String.format("Incorrect Grouper %s used for KinesisSystemConsumer %s. Please set the %s config" + " to %s.", jobConfig.getSystemStreamPartitionGrouperFactory(), system, JobConfig.SSP_GROUPER_FACTORY, AllSspToSingleTaskGrouperFactory.class.getCanonicalName());
        throw new ConfigException(errMsg);
    }
    // Kinesis streams cannot be configured as broadcast streams
    TaskConfig taskConfig = new TaskConfig(config);
    if (taskConfig.getBroadcastSystemStreams().stream().anyMatch(ss -> system.equals(ss.getSystem()))) {
        throw new ConfigException("Kinesis streams cannot be configured as broadcast streams.");
    }
    // Kinesis streams cannot be configured as bootstrap streams
    KinesisConfig kConfig = new KinesisConfig(config);
    kConfig.getKinesisStreams(system).forEach(stream -> {
        StreamConfig streamConfig = new StreamConfig(kConfig);
        SystemStream ss = new SystemStream(system, stream);
        if (streamConfig.getBootstrapEnabled(ss)) {
            throw new ConfigException("Kinesis streams cannot be configured as bootstrap streams.");
        }
    });
}

Example 23 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestStreamUtil method testGetStreamWithOutSystemInConfig.

// System is required. Throw if it cannot be determined.
@Test(expected = IllegalArgumentException.class)
public void testGetStreamWithOutSystemInConfig() {
    Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME);
    StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
    assertEquals(TEST_SYSTEM, spec.getSystemName());
}

Example 24 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestStreamUtil method testGetStreamWithSystemAtBothScopesInConfig.

// Stream scope should override default scope
@Test
public void testGetStreamWithSystemAtBothScopesInConfig() {
    Config config = addConfigs(buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, StreamConfig.SYSTEM, TEST_SYSTEM), JobConfig.JOB_DEFAULT_SYSTEM, TEST_DEFAULT_SYSTEM);
    StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
    assertEquals(TEST_SYSTEM, spec.getSystemName());
}

Example 25 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestStreamUtil method testGetStreamPhysicalNameArgSpecialCharacters.

// Special characters are allowed for the physical name
@Test
public void testGetStreamPhysicalNameArgSpecialCharacters() {
    Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME_SPECIAL_CHARS, StreamConfig.SYSTEM, TEST_SYSTEM);
    StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
    assertEquals(TEST_PHYSICAL_NAME_SPECIAL_CHARS, spec.getPhysicalName());
}