Examples with ZkUtils kafka.utils.ZkUtils used on opensource projects

Example 41 with ZkUtils

use of kafka.utils.ZkUtils in project incubator-rya by apache.

the class KafkaTopics method createTopics.

/**
 * Creates a set of Kafka topics for each topic that does not already exist.
 *
 * @param zookeeperServers - The Zookeeper servers that are used by the Kafka Streams program. (not null)
 * @param topicNames - The topics that will be created. (not null)
 * @param partitions - The number of partitions that each of the topics will have.
 * @param replicationFactor - The replication factor of the topics that are created.
 */
public static void createTopics(final String zookeeperServers, final Set<String> topicNames, final int partitions, final int replicationFactor) {
    requireNonNull(zookeeperServers);
    requireNonNull(topicNames);
    ZkUtils zkUtils = null;
    try {
        zkUtils = ZkUtils.apply(new ZkClient(zookeeperServers, 30000, 30000, ZKStringSerializer$.MODULE$), false);
        for (final String topicName : topicNames) {
            if (!AdminUtils.topicExists(zkUtils, topicName)) {
                AdminUtils.createTopic(zkUtils, topicName, partitions, replicationFactor, new Properties(), RackAwareMode.Disabled$.MODULE$);
            }
        }
    } finally {
        if (zkUtils != null) {
            zkUtils.close();
        }
    }
}

Example 42 with ZkUtils

use of kafka.utils.ZkUtils in project atlas by apache.

the class AtlasTopicCreatorTest method shouldCreateTopicEvenIfEarlierOneFails.

@Test
public void shouldCreateTopicEvenIfEarlierOneFails() {
    Configuration configuration = mock(Configuration.class);
    when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(true);
    when(configuration.getString("atlas.authentication.method.kerberos")).thenReturn("false");
    final ZkUtils zookeeperUtils = mock(ZkUtils.class);
    final Map<String, Boolean> createdTopics = new HashMap<>();
    createdTopics.put("ATLAS_ENTITIES", false);
    AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator() {

        @Override
        protected boolean ifTopicExists(String topicName, ZkUtils zkUtils) {
            return false;
        }

        @Override
        protected ZkUtils createZkUtils(Configuration atlasProperties) {
            return zookeeperUtils;
        }

        @Override
        protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) {
            if (topicName.equals("ATLAS_HOOK")) {
                throw new RuntimeException("Simulating failure when creating ATLAS_HOOK topic");
            } else {
                createdTopics.put(topicName, true);
            }
        }
    };
    atlasTopicCreator.createAtlasTopic(configuration, "ATLAS_HOOK", "ATLAS_ENTITIES");
    assertTrue(createdTopics.get("ATLAS_ENTITIES"));
}

Example 43 with ZkUtils

use of kafka.utils.ZkUtils in project atlas by apache.

the class AtlasTopicCreatorTest method shouldCreateTopicIfItDoesNotExist.

@Test
public void shouldCreateTopicIfItDoesNotExist() {
    Configuration configuration = mock(Configuration.class);
    when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(true);
    when(configuration.getString("atlas.authentication.method.kerberos")).thenReturn("false");
    final ZkUtils zookeeperUtils = mock(ZkUtils.class);
    final boolean[] createdTopic = new boolean[] { false };
    AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator() {

        @Override
        protected boolean ifTopicExists(String topicName, ZkUtils zkUtils) {
            return false;
        }

        @Override
        protected ZkUtils createZkUtils(Configuration atlasProperties) {
            return zookeeperUtils;
        }

        @Override
        protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) {
            createdTopic[0] = true;
        }
    };
    atlasTopicCreator.createAtlasTopic(configuration, "ATLAS_HOOK");
    assertTrue(createdTopic[0]);
}

Example 44 with ZkUtils

use of kafka.utils.ZkUtils in project atlas by apache.

the class AtlasTopicCreatorTest method shouldCreateMultipleTopics.

@Test
public void shouldCreateMultipleTopics() {
    Configuration configuration = mock(Configuration.class);
    when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(true);
    when(configuration.getString("atlas.authentication.method.kerberos")).thenReturn("false");
    final ZkUtils zookeeperUtils = mock(ZkUtils.class);
    final Map<String, Boolean> createdTopics = new HashMap<>();
    createdTopics.put("ATLAS_HOOK", false);
    createdTopics.put("ATLAS_ENTITIES", false);
    AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator() {

        @Override
        protected boolean ifTopicExists(String topicName, ZkUtils zkUtils) {
            return false;
        }

        @Override
        protected ZkUtils createZkUtils(Configuration atlasProperties) {
            return zookeeperUtils;
        }

        @Override
        protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) {
            createdTopics.put(topicName, true);
        }
    };
    atlasTopicCreator.createAtlasTopic(configuration, "ATLAS_HOOK", "ATLAS_ENTITIES");
    assertTrue(createdTopics.get("ATLAS_HOOK"));
    assertTrue(createdTopics.get("ATLAS_ENTITIES"));
}

Example 45 with ZkUtils

use of kafka.utils.ZkUtils in project testcases by coheigea.

the class KafkaAuthorizerSASLGSSTest method setup.

@org.junit.BeforeClass
public static void setup() throws Exception {
    String basedir = System.getProperty("basedir");
    if (basedir == null) {
        basedir = new File(".").getCanonicalPath();
    }
    configureKerby(basedir);
    // JAAS Config file - We need to point to the correct keytab files
    Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/kafka_kerberos.jaas");
    String content = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
    content = content.replaceAll("<basedir>", basedir);
    Path path2 = FileSystems.getDefault().getPath(basedir, "/target/test-classes/kafka_kerberos.jaas");
    Files.write(path2, content.getBytes(StandardCharsets.UTF_8));
    System.setProperty("java.security.auth.login.config", path2.toString());
    // Set up Zookeeper to require SASL
    Map<String, Object> zookeeperProperties = new HashMap<>();
    zookeeperProperties.put("authProvider.1", "org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
    zookeeperProperties.put("requireClientAuthScheme", "sasl");
    zookeeperProperties.put("jaasLoginRenew", "3600000");
    InstanceSpec instanceSpec = new InstanceSpec(null, -1, -1, -1, true, 1, -1, -1, zookeeperProperties);
    zkServer = new TestingServer(instanceSpec, true);
    // Get a random port
    ServerSocket serverSocket = new ServerSocket(0);
    port = serverSocket.getLocalPort();
    serverSocket.close();
    final Properties props = new Properties();
    props.put("broker.id", 1);
    props.put("host.name", "localhost");
    props.put("port", port);
    props.put("log.dir", "/tmp/kafka");
    props.put("zookeeper.connect", zkServer.getConnectString());
    props.put("replica.socket.timeout.ms", "1500");
    props.put("controlled.shutdown.enable", Boolean.TRUE.toString());
    // Enable SASL_PLAINTEXT
    props.put("listeners", "SASL_PLAINTEXT://localhost:" + port);
    props.put("security.inter.broker.protocol", "SASL_PLAINTEXT");
    props.put("sasl.enabled.mechanisms", "GSSAPI");
    props.put("sasl.mechanism.inter.broker.protocol", "GSSAPI");
    props.put("sasl.kerberos.service.name", "kafka");
    // Plug in custom authorizer
    props.put("authorizer.class.name", "org.apache.coheigea.bigdata.kafka.CustomSASLGSSAuthorizer");
    KafkaConfig config = new KafkaConfig(props);
    kafkaServer = new KafkaServerStartable(config);
    kafkaServer.startup();
    // Create some topics
    ZkClient zkClient = new ZkClient(zkServer.getConnectString(), 30000, 30000, ZKStringSerializer$.MODULE$);
    final ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkServer.getConnectString()), false);
    AdminUtils.createTopic(zkUtils, "test", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
    AdminUtils.createTopic(zkUtils, "dev", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
}