Examples with KafkaSystemAdmin org.apache.samza.system.kafka.KafkaSystemAdmin used on opensource projects

Example 1 with KafkaSystemAdmin

use of org.apache.samza.system.kafka.KafkaSystemAdmin in project samza by apache.

the class TestKafkaSystemAdminJava method testCreateStream.

@Test
public void testCreateStream() {
    StreamSpec spec = new StreamSpec("testId", "testStream", "testSystem", 8);
    KafkaSystemAdmin admin = systemAdmin();
    assertTrue("createStream should return true if the stream does not exist and then is created.", admin.createStream(spec));
    admin.validateStream(spec);
    assertFalse("createStream should return false if the stream already exists.", systemAdmin().createStream(spec));
}

Example 2 with KafkaSystemAdmin

use of org.apache.samza.system.kafka.KafkaSystemAdmin in project samza by apache.

the class TestKafkaSystemAdminJava method testToKafkaSpecForCheckpointStreamShouldReturnTheCorrectStreamSpecByPreservingTheConfig.

@Test
public void testToKafkaSpecForCheckpointStreamShouldReturnTheCorrectStreamSpecByPreservingTheConfig() {
    String topicName = "testStream";
    String streamId = "samza-internal-checkpoint-stream-id";
    int partitionCount = 1;
    Map<String, String> map = new HashMap<>();
    map.put("cleanup.policy", "compact");
    map.put("replication.factor", "3");
    map.put("segment.bytes", "536870912");
    map.put("delete.retention.ms", "86400000");
    Config config = new MapConfig(map);
    StreamSpec spec = new StreamSpec(streamId, topicName, SYSTEM, partitionCount, config);
    KafkaSystemAdmin kafkaSystemAdmin = systemAdmin();
    KafkaStreamSpec kafkaStreamSpec = kafkaSystemAdmin.toKafkaSpec(spec);
    System.out.println(kafkaStreamSpec);
    assertEquals(streamId, kafkaStreamSpec.getId());
    assertEquals(topicName, kafkaStreamSpec.getPhysicalName());
    assertEquals(partitionCount, kafkaStreamSpec.getPartitionCount());
    assertEquals(3, kafkaStreamSpec.getReplicationFactor());
    assertEquals("compact", kafkaStreamSpec.getConfig().get("cleanup.policy"));
    assertEquals("536870912", kafkaStreamSpec.getConfig().get("segment.bytes"));
    assertEquals("86400000", kafkaStreamSpec.getConfig().get("delete.retention.ms"));
}

Example 3 with KafkaSystemAdmin

use of org.apache.samza.system.kafka.KafkaSystemAdmin in project samza by apache.

the class TestKafkaSystemAdminJava method testClearStream.

@Test
public void testClearStream() {
    StreamSpec spec = new StreamSpec("testId", "testStreamClear", "testSystem", 8);
    KafkaSystemAdmin admin = systemAdmin();
    String topicName = spec.getPhysicalName();
    assertTrue("createStream should return true if the stream does not exist and then is created.", admin.createStream(spec));
    // validate topic exists
    assertTrue(admin.clearStream(spec));
    // validate that topic was removed
    DescribeTopicsResult dtr = admin.adminClient.describeTopics(ImmutableSet.of(topicName));
    try {
        TopicDescription td = dtr.all().get().get(topicName);
        Assert.fail("topic " + topicName + " should've been removed. td=" + td);
    } catch (Exception e) {
        if (!(e.getCause() instanceof org.apache.kafka.common.errors.UnknownTopicOrPartitionException)) {
            Assert.fail("topic " + topicName + " should've been removed. Expected UnknownTopicOrPartitionException.");
        }
    }
}

Example 4 with KafkaSystemAdmin

use of org.apache.samza.system.kafka.KafkaSystemAdmin in project samza by apache.

the class IntegrationTestHarness method createSystemAdmin.

private KafkaSystemAdmin createSystemAdmin(String system) {
    String kafkaConsumerPropertyPrefix = "systems." + system + ".consumer.";
    Map<String, String> map = new HashMap<>();
    map.put(kafkaConsumerPropertyPrefix + CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerList());
    map.put(JobConfig.JOB_NAME, "test.job");
    Config config = new MapConfig(map);
    HashMap<String, Object> consumerConfig = KafkaConsumerConfig.getKafkaSystemConsumerConfig(config, system, KafkaConsumerConfig.createClientId("kafka-admin-consumer", config));
    return new KafkaSystemAdmin(system, new MapConfig(map), KafkaSystemConsumer.createKafkaConsumerImpl(system, consumerConfig));
}

Example 5 with KafkaSystemAdmin

use of org.apache.samza.system.kafka.KafkaSystemAdmin in project samza by apache.

the class TestKafkaSystemAdminJava method testCreateChangelogStreamHelp.

public void testCreateChangelogStreamHelp(final String topic) {
    final int partitions = 12;
    final int repFactor = 2;
    Map<String, String> map = new HashMap<>();
    map.put(JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM);
    map.put(String.format("stores.%s.changelog", "fakeStore"), topic);
    map.put(String.format("stores.%s.changelog.replication.factor", "fakeStore"), String.valueOf(repFactor));
    map.put(String.format("stores.%s.changelog.kafka.segment.bytes", "fakeStore"), "139");
    KafkaSystemAdmin admin = Mockito.spy(createSystemAdmin(SYSTEM, map));
    StreamSpec spec = StreamSpec.createChangeLogStreamSpec(topic, SYSTEM, partitions);
    Mockito.doAnswer(invocationOnMock -> {
        StreamSpec internalSpec = (StreamSpec) invocationOnMock.callRealMethod();
        // KafkaStreamSpec is used to carry replication factor
        assertTrue(internalSpec instanceof KafkaStreamSpec);
        assertTrue(internalSpec.isChangeLogStream());
        assertEquals(SYSTEM, internalSpec.getSystemName());
        assertEquals(topic, internalSpec.getPhysicalName());
        assertEquals(repFactor, ((KafkaStreamSpec) internalSpec).getReplicationFactor());
        assertEquals(partitions, internalSpec.getPartitionCount());
        assertEquals("139", ((KafkaStreamSpec) internalSpec).getProperties().getProperty("segment.bytes"));
        assertEquals("compact", ((KafkaStreamSpec) internalSpec).getProperties().getProperty("cleanup.policy"));
        return internalSpec;
    }).when(admin).toKafkaSpec(Mockito.any());
    admin.createStream(spec);
    admin.validateStream(spec);
}