Examples with ZkUtils kafka.utils.ZkUtils used on opensource projects

Example 51 with ZkUtils

use of kafka.utils.ZkUtils in project testcases by coheigea.

the class KafkaTest method testKafka.

@org.junit.Test
public void testKafka() throws Exception {
    // Create a "test" topic
    ZkClient zkClient = new ZkClient(zkServer.getConnectString(), 30000, 30000, ZKStringSerializer$.MODULE$);
    final ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkServer.getConnectString()), false);
    AdminUtils.createTopic(zkUtils, "test", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
    // Create the Producer
    Properties producerProps = new Properties();
    producerProps.put("bootstrap.servers", "localhost:" + port);
    producerProps.put("acks", "all");
    producerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    producerProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    Producer<String, String> producer = new KafkaProducer<>(producerProps);
    // Create the Consumer
    Properties consumerProps = new Properties();
    consumerProps.put("bootstrap.servers", "localhost:" + port);
    consumerProps.put("group.id", "test");
    consumerProps.put("auto.offset.reset", "earliest");
    consumerProps.put("enable.auto.commit", "true");
    consumerProps.put("auto.commit.interval.ms", "1000");
    consumerProps.put("session.timeout.ms", "30000");
    consumerProps.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    consumerProps.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(consumerProps);
    consumer.subscribe(Arrays.asList("test"));
    // Send a message
    producer.send(new ProducerRecord<String, String>("test", "somekey", "somevalue"));
    producer.flush();
    // Poll until we consume it
    ConsumerRecord<String, String> record = null;
    for (int i = 0; i < 1000; i++) {
        ConsumerRecords<String, String> records = consumer.poll(100);
        if (records.count() > 0) {
            record = records.iterator().next();
            break;
        }
        Thread.sleep(1000);
    }
    Assert.assertNotNull(record);
    Assert.assertEquals("somevalue", record.value());
    producer.close();
    consumer.close();
}

Example 52 with ZkUtils

use of kafka.utils.ZkUtils in project testcases by coheigea.

the class KafkaRangerAuthorizerTest method setup.

@org.junit.BeforeClass
public static void setup() throws Exception {
    zkServer = new TestingServer();
    // Get a random port
    ServerSocket serverSocket = new ServerSocket(0);
    port = serverSocket.getLocalPort();
    serverSocket.close();
    final Properties props = new Properties();
    props.put("broker.id", 1);
    props.put("host.name", "localhost");
    props.put("port", port);
    props.put("log.dir", "/tmp/kafka");
    props.put("zookeeper.connect", zkServer.getConnectString());
    props.put("replica.socket.timeout.ms", "1500");
    props.put("controlled.shutdown.enable", Boolean.TRUE.toString());
    // Enable SSL
    props.put("listeners", "SSL://localhost:" + port);
    props.put("ssl.keystore.location", KafkaAuthorizerTest.class.getResource("/servicestore.jks").getPath());
    props.put("ssl.keystore.password", "sspass");
    props.put("ssl.key.password", "skpass");
    props.put("ssl.truststore.location", KafkaAuthorizerTest.class.getResource("/truststore.jks").getPath());
    props.put("ssl.truststore.password", "security");
    props.put("security.inter.broker.protocol", "SSL");
    props.put("ssl.client.auth", "required");
    // Plug in Apache Ranger authorizer
    props.put("authorizer.class.name", "org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer");
    // Create users for testing
    UserGroupInformation.createUserForTesting("CN=Client,O=Apache,L=Dublin,ST=Leinster,C=IE", new String[] { "public" });
    UserGroupInformation.createUserForTesting("CN=Service,O=Apache,L=Dublin,ST=Leinster,C=IE", new String[] { "IT" });
    KafkaConfig config = new KafkaConfig(props);
    kafkaServer = new KafkaServerStartable(config);
    kafkaServer.startup();
    // Create some topics
    ZkClient zkClient = new ZkClient(zkServer.getConnectString(), 30000, 30000, ZKStringSerializer$.MODULE$);
    final ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkServer.getConnectString()), false);
    AdminUtils.createTopic(zkUtils, "test", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
    AdminUtils.createTopic(zkUtils, "dev", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
    AdminUtils.createTopic(zkUtils, "messages", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
}

Example 53 with ZkUtils

use of kafka.utils.ZkUtils in project testcases by coheigea.

the class KafkaSentryAuthorizerTest method setup.

@org.junit.BeforeClass
public static void setup() throws Exception {
    zkServer = new TestingServer();
    // Get a random port
    ServerSocket serverSocket = new ServerSocket(0);
    port = serverSocket.getLocalPort();
    serverSocket.close();
    final Properties props = new Properties();
    props.put("broker.id", 1);
    props.put("host.name", "localhost");
    props.put("port", port);
    props.put("log.dir", "/tmp/kafka");
    props.put("zookeeper.connect", zkServer.getConnectString());
    props.put("replica.socket.timeout.ms", "1500");
    props.put("controlled.shutdown.enable", Boolean.TRUE.toString());
    // Enable SSL
    props.put("listeners", "SSL://localhost:" + port);
    props.put("ssl.keystore.location", KafkaAuthorizerTest.class.getResource("/servicestore.jks").getPath());
    props.put("ssl.keystore.password", "sspass");
    props.put("ssl.key.password", "skpass");
    props.put("ssl.truststore.location", KafkaAuthorizerTest.class.getResource("/truststore.jks").getPath());
    props.put("ssl.truststore.password", "security");
    props.put("security.inter.broker.protocol", "SSL");
    props.put("ssl.client.auth", "required");
    // Plug in Apache Sentry authorizer
    props.put("authorizer.class.name", "org.apache.sentry.kafka.authorizer.SentryKafkaAuthorizer");
    props.put("sentry.kafka.site.url", "file:" + KafkaAuthorizerTest.class.getResource("/sentry-site.xml").getPath());
    KafkaConfig config = new KafkaConfig(props);
    kafkaServer = new KafkaServerStartable(config);
    kafkaServer.startup();
    // Create some topics
    ZkClient zkClient = new ZkClient(zkServer.getConnectString(), 30000, 30000, ZKStringSerializer$.MODULE$);
    final ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkServer.getConnectString()), false);
    AdminUtils.createTopic(zkUtils, "test", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
    AdminUtils.createTopic(zkUtils, "dev", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
}

Example 54 with ZkUtils

use of kafka.utils.ZkUtils in project cruise-control by linkedin.

the class ExecutorTest method testMoveNonExistingPartition.

@Test
public void testMoveNonExistingPartition() throws InterruptedException {
    ZkUtils zkUtils = KafkaCruiseControlUnitTestUtils.zkUtils(zookeeper().getConnectionString());
    AdminClient adminClient = getAdminClient(broker(0).getPlaintextAddr());
    adminClient.createTopics(Arrays.asList(new NewTopic(TOPIC_0, 1, (short) 1), new NewTopic(TOPIC_1, 1, (short) 2)));
    Map<String, TopicDescription> topicDescriptions = createTopics();
    int initialLeader0 = topicDescriptions.get(TOPIC_0).partitions().get(0).leader().id();
    int initialLeader1 = topicDescriptions.get(TOPIC_1).partitions().get(0).leader().id();
    ExecutionProposal proposal0 = new ExecutionProposal(TP0, 0, initialLeader0, Collections.singletonList(initialLeader0), Collections.singletonList(initialLeader0 == 0 ? 1 : 0));
    ExecutionProposal proposal1 = new ExecutionProposal(TP1, 0, initialLeader1, Arrays.asList(initialLeader1, initialLeader1 == 0 ? 1 : 0), Arrays.asList(initialLeader1 == 0 ? 1 : 0, initialLeader1));
    ExecutionProposal proposal2 = new ExecutionProposal(TP2, 0, initialLeader0, Collections.singletonList(initialLeader0), Collections.singletonList(initialLeader0 == 0 ? 1 : 0));
    ExecutionProposal proposal3 = new ExecutionProposal(TP3, 0, initialLeader1, Arrays.asList(initialLeader1, initialLeader1 == 0 ? 1 : 0), Arrays.asList(initialLeader1 == 0 ? 1 : 0, initialLeader1));
    Collection<ExecutionProposal> proposalsToExecute = Arrays.asList(proposal0, proposal1, proposal2, proposal3);
    Collection<ExecutionProposal> proposalsToCheck = Arrays.asList(proposal0, proposal1);
    executeAndVerifyProposals(zkUtils, proposalsToExecute, proposalsToCheck);
}

Example 55 with ZkUtils

use of kafka.utils.ZkUtils in project cruise-control by linkedin.

the class ExecutorTest method testBasicBalanceMovement.

@Test
public void testBasicBalanceMovement() {
    ZkUtils zkUtils = KafkaCruiseControlUnitTestUtils.zkUtils(zookeeper().getConnectionString());
    String topic0 = "testPartitionMovement0";
    String topic1 = "testPartitionMovement1";
    int partition = 0;
    TopicPartition tp0 = new TopicPartition(topic0, partition);
    TopicPartition tp1 = new TopicPartition(topic1, partition);
    AdminUtils.createTopic(zkUtils, topic0, 1, 1, new Properties(), RackAwareMode.Safe$.MODULE$);
    AdminUtils.createTopic(zkUtils, topic1, 1, 2, new Properties(), RackAwareMode.Safe$.MODULE$);
    while (zkUtils.getLeaderForPartition(topic0, partition).isEmpty() || zkUtils.getLeaderForPartition(topic1, partition).isEmpty()) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    int initialLeader0 = (Integer) zkUtils.getLeaderForPartition(topic0, partition).get();
    int initialLeader1 = (Integer) zkUtils.getLeaderForPartition(topic1, partition).get();
    ExecutionProposal proposal0 = new ExecutionProposal(tp0, 0, initialLeader0, Collections.singletonList(initialLeader0), Collections.singletonList(initialLeader0 == 0 ? 1 : 0));
    ExecutionProposal proposal1 = new ExecutionProposal(tp1, 0, initialLeader1, Arrays.asList(initialLeader1, initialLeader1 == 0 ? 1 : 0), Arrays.asList(initialLeader1 == 0 ? 1 : 0, initialLeader1));
    Collection<ExecutionProposal> proposals = Arrays.asList(proposal0, proposal1);
    executeAndVerifyProposals(zkUtils, proposals, proposals);
}