Examples with MapConfig org.apache.samza.config.MapConfig used on opensource projects

Example 26 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestGroupBySystemStreamPartition method testElasticityEnabledLocalStreamGroupedCorrectly.

@Test
public void testElasticityEnabledLocalStreamGroupedCorrectly() {
    Config config = new MapConfig(ImmutableMap.of("job.elasticity.factor", "2"));
    SystemStreamPartitionGrouper grouper = grouperFactory.getSystemStreamPartitionGrouper(config);
    Map<TaskName, Set<SystemStreamPartition>> emptyResult = grouper.group(new HashSet<>());
    assertTrue(emptyResult.isEmpty());
    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(ImmutableSet.of(aa0, aa1, aa2, ac0));
    Map<TaskName, Set<SystemStreamPartition>> expectedResult = ImmutableMap.<TaskName, Set<SystemStreamPartition>>builder().put(new TaskName(new SystemStreamPartition(aa0, 0).toString()), ImmutableSet.of(new SystemStreamPartition(aa0, 0))).put(new TaskName(new SystemStreamPartition(aa0, 1).toString()), ImmutableSet.of(new SystemStreamPartition(aa0, 1))).put(new TaskName(new SystemStreamPartition(aa1, 0).toString()), ImmutableSet.of(new SystemStreamPartition(aa1, 0))).put(new TaskName(new SystemStreamPartition(aa1, 1).toString()), ImmutableSet.of(new SystemStreamPartition(aa1, 1))).put(new TaskName(new SystemStreamPartition(aa2, 0).toString()), ImmutableSet.of(new SystemStreamPartition(aa2, 0))).put(new TaskName(new SystemStreamPartition(aa2, 1).toString()), ImmutableSet.of(new SystemStreamPartition(aa2, 1))).put(new TaskName(new SystemStreamPartition(ac0, 0).toString()), ImmutableSet.of(new SystemStreamPartition(ac0, 0))).put(new TaskName(new SystemStreamPartition(ac0, 1).toString()), ImmutableSet.of(new SystemStreamPartition(ac0, 1))).build();
    assertEquals(expectedResult, result);
}

Example 27 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestKafkaCheckpointManagerFactory method testGetCheckpointTopicProperties.

@Test
public void testGetCheckpointTopicProperties() {
    Map<String, String> config = new HashMap<>();
    Properties properties = new KafkaConfig(new MapConfig(config)).getCheckpointTopicProperties();
    assertEquals(properties.getProperty("cleanup.policy"), "compact");
    assertEquals(properties.getProperty("segment.bytes"), String.valueOf(KafkaConfig.DEFAULT_CHECKPOINT_SEGMENT_BYTES()));
    config.put(ApplicationConfig.APP_MODE, ApplicationConfig.ApplicationMode.BATCH.name());
    properties = new KafkaConfig(new MapConfig(config)).getCheckpointTopicProperties();
    assertEquals(properties.getProperty("cleanup.policy"), "compact,delete");
    assertEquals(properties.getProperty("segment.bytes"), String.valueOf(KafkaConfig.DEFAULT_CHECKPOINT_SEGMENT_BYTES()));
    assertEquals(properties.getProperty("retention.ms"), String.valueOf(KafkaConfig.DEFAULT_RETENTION_MS_FOR_BATCH()));
}

Example 28 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestKafkaSystemAdminJava method testToKafkaSpecForCheckpointStreamShouldReturnTheCorrectStreamSpecByPreservingTheConfig.

@Test
public void testToKafkaSpecForCheckpointStreamShouldReturnTheCorrectStreamSpecByPreservingTheConfig() {
    String topicName = "testStream";
    String streamId = "samza-internal-checkpoint-stream-id";
    int partitionCount = 1;
    Map<String, String> map = new HashMap<>();
    map.put("cleanup.policy", "compact");
    map.put("replication.factor", "3");
    map.put("segment.bytes", "536870912");
    map.put("delete.retention.ms", "86400000");
    Config config = new MapConfig(map);
    StreamSpec spec = new StreamSpec(streamId, topicName, SYSTEM, partitionCount, config);
    KafkaSystemAdmin kafkaSystemAdmin = systemAdmin();
    KafkaStreamSpec kafkaStreamSpec = kafkaSystemAdmin.toKafkaSpec(spec);
    System.out.println(kafkaStreamSpec);
    assertEquals(streamId, kafkaStreamSpec.getId());
    assertEquals(topicName, kafkaStreamSpec.getPhysicalName());
    assertEquals(partitionCount, kafkaStreamSpec.getPartitionCount());
    assertEquals(3, kafkaStreamSpec.getReplicationFactor());
    assertEquals("compact", kafkaStreamSpec.getConfig().get("cleanup.policy"));
    assertEquals("536870912", kafkaStreamSpec.getConfig().get("segment.bytes"));
    assertEquals("86400000", kafkaStreamSpec.getConfig().get("delete.retention.ms"));
}

Example 29 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestKafkaSystemFactoryJava method testGetIntermediateStreamProperties.

@Test
public void testGetIntermediateStreamProperties() {
    Map<String, String> config = new HashMap<>();
    KafkaSystemFactory factory = new KafkaSystemFactory();
    Map<String, Properties> properties = JavaConversions.mapAsJavaMap(factory.getIntermediateStreamProperties(new MapConfig(config)));
    assertTrue(properties.isEmpty());
    // no properties for stream
    config.put("streams.test.samza.intermediate", "true");
    // some random config
    config.put("streams.test.compression.type", "lz4");
    properties = JavaConversions.mapAsJavaMap(factory.getIntermediateStreamProperties(new MapConfig(config)));
    assertTrue(properties.isEmpty());
    config.put(ApplicationConfig.APP_MODE, ApplicationConfig.ApplicationMode.BATCH.name());
    KafkaSystemAdmin admin = createSystemAdmin(SYSTEM(), config);
    StreamSpec spec = new StreamSpec("test", "test", SYSTEM(), Collections.singletonMap("replication.factor", "1"));
    KafkaStreamSpec kspec = admin.toKafkaSpec(spec);
    Properties prop = kspec.getProperties();
    assertEquals(prop.getProperty("retention.ms"), String.valueOf(KafkaConfig.DEFAULT_RETENTION_MS_FOR_BATCH()));
    assertEquals(prop.getProperty("compression.type"), "lz4");
    // replication.factor should be removed from the properties and set on the spec directly
    assertEquals(kspec.getReplicationFactor(), 1);
    assertNull(prop.getProperty("replication.factor"));
}

Example 30 with MapConfig

use of org.apache.samza.config.MapConfig in project samza by apache.

the class TestBaseKeyValueStorageEngineFactory method testMissingKeySerde.

@Test(expected = SamzaException.class)
public void testMissingKeySerde() {
    Config config = new MapConfig(BASE_CONFIG);
    when(this.jobContext.getConfig()).thenReturn(config);
    new MockKeyValueStorageEngineFactory(this.rawKeyValueStore).getStorageEngine(STORE_NAME, this.storeDir, null, this.msgSerde, this.changelogCollector, this.metricsRegistry, null, this.jobContext, this.containerContext, STORE_MODE);
}