Examples with StreamConfig org.apache.samza.config.StreamConfig used on opensource projects

Example 31 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestStreamUtil method testGetStreamSystemNameArgValid.

// When the system name is provided explicitly, it should be used, regardless of whether it's also in the config
@Test
public void testGetStreamSystemNameArgValid() {
    Config config = buildStreamConfig(STREAM_ID, // This should be ignored because of the explicit arg
    StreamConfig.PHYSICAL_NAME, // This should be ignored because of the explicit arg
    TEST_PHYSICAL_NAME, StreamConfig.SYSTEM, // This too
    TEST_SYSTEM);
    StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
    assertEquals(STREAM_ID, spec.getId());
    assertEquals(TEST_PHYSICAL_NAME, spec.getPhysicalName());
    assertEquals(TEST_SYSTEM, spec.getSystemName());
}

Example 32 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestStreamUtil method testGetStreamStreamIdInvalid.

// Special characters are NOT allowed for streamId, because it's used as an identifier in the config.
@Test(expected = IllegalArgumentException.class)
public void testGetStreamStreamIdInvalid() {
    Config config = buildStreamConfig(STREAM_ID_INVALID, StreamConfig.SYSTEM, TEST_SYSTEM);
    StreamUtil.getStreamSpec(STREAM_ID_INVALID, new StreamConfig(config));
}

Example 33 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestStreamUtil method testGetStreamSystemNameArgEmpty.

// Empty strings are NOT allowed for system name, because it's used as an identifier in the config.
@Test(expected = IllegalArgumentException.class)
public void testGetStreamSystemNameArgEmpty() {
    Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, StreamConfig.SYSTEM, "");
    StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
}

Example 34 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class TestStreamUtil method testStreamConfigOverridesWithSystemDefaults.

// Verify that we use a default specified with systems.x.default.stream.*, if specified
@Test
public void testStreamConfigOverridesWithSystemDefaults() {
    Config config = addConfigs(buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, StreamConfig.SYSTEM, TEST_SYSTEM, "segment.bytes", "5309"), // System default property
    String.format("systems.%s.default.stream.replication.factor", TEST_SYSTEM), // System default property
    "4", String.format("systems.%s.default.stream.segment.bytest", TEST_SYSTEM), "867");
    StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config));
    Map<String, String> properties = spec.getConfig();
    assertEquals(3, properties.size());
    // Uses system default
    assertEquals("4", properties.get("replication.factor"));
    // Overrides system default
    assertEquals("5309", properties.get("segment.bytes"));
}

Example 35 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class OperatorImplGraph method computeOutputToInput.

private static void computeOutputToInput(SystemStream input, OperatorSpec opSpec, Multimap<SystemStream, SystemStream> outputToInputStreams, StreamConfig streamConfig) {
    if (opSpec instanceof PartitionByOperatorSpec) {
        PartitionByOperatorSpec spec = (PartitionByOperatorSpec) opSpec;
        SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId());
        outputToInputStreams.put(systemStream, input);
    } else if (opSpec instanceof BroadcastOperatorSpec) {
        BroadcastOperatorSpec spec = (BroadcastOperatorSpec) opSpec;
        SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId());
        outputToInputStreams.put(systemStream, input);
    } else {
        Collection<OperatorSpec> nextOperators = opSpec.getRegisteredOperatorSpecs();
        nextOperators.forEach(spec -> computeOutputToInput(input, spec, outputToInputStreams, streamConfig));
    }
}