Search in sources :

Example 21 with ZkUtils

use of kafka.utils.ZkUtils in project atlas by apache.

the class AtlasTopicCreatorTest method shouldNotCreateAtlasTopicIfNotConfiguredToDoSo.

@Test
public void shouldNotCreateAtlasTopicIfNotConfiguredToDoSo() {
    Configuration configuration = mock(Configuration.class);
    when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(false);
    when(configuration.getString("atlas.authentication.method.kerberos")).thenReturn("false");
    final boolean[] topicExistsCalled = new boolean[] { false };
    AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator() {

        @Override
        protected boolean ifTopicExists(String topicName, ZkUtils zkUtils) {
            topicExistsCalled[0] = true;
            return false;
        }
    };
    atlasTopicCreator.createAtlasTopic(configuration, "ATLAS_HOOK");
    assertFalse(topicExistsCalled[0]);
}
Also used : Configuration(org.apache.commons.configuration.Configuration) ZkUtils(kafka.utils.ZkUtils) Test(org.testng.annotations.Test)

Example 22 with ZkUtils

use of kafka.utils.ZkUtils in project atlas by apache.

the class AtlasTopicCreatorTest method shouldNotFailIfExceptionOccursDuringCreatingTopic.

@Test
public void shouldNotFailIfExceptionOccursDuringCreatingTopic() {
    Configuration configuration = mock(Configuration.class);
    when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(true);
    when(configuration.getString("atlas.authentication.method.kerberos")).thenReturn("false");
    final ZkUtils zookeeperUtils = mock(ZkUtils.class);
    final boolean[] createTopicCalled = new boolean[] { false };
    AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator() {

        @Override
        protected boolean ifTopicExists(String topicName, ZkUtils zkUtils) {
            return false;
        }

        @Override
        protected ZkUtils createZkUtils(Configuration atlasProperties) {
            return zookeeperUtils;
        }

        @Override
        protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) {
            createTopicCalled[0] = true;
            throw new RuntimeException("Simulating failure during creating topic");
        }
    };
    atlasTopicCreator.createAtlasTopic(configuration, "ATLAS_HOOK");
    assertTrue(createTopicCalled[0]);
}
Also used : Configuration(org.apache.commons.configuration.Configuration) ZkUtils(kafka.utils.ZkUtils) Test(org.testng.annotations.Test)

Example 23 with ZkUtils

use of kafka.utils.ZkUtils in project atlas by apache.

the class AtlasTopicCreatorTest method shouldNotProcessTopicCreationIfSecurityFails.

@Test
public void shouldNotProcessTopicCreationIfSecurityFails() {
    Configuration configuration = mock(Configuration.class);
    when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(true);
    final ZkUtils zookeeperUtils = mock(ZkUtils.class);
    final Map<String, Boolean> createdTopics = new HashMap<>();
    createdTopics.put("ATLAS_HOOK", false);
    createdTopics.put("ATLAS_ENTITIES", false);
    AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator() {

        @Override
        protected boolean ifTopicExists(String topicName, ZkUtils zkUtils) {
            return false;
        }

        @Override
        protected ZkUtils createZkUtils(Configuration atlasProperties) {
            return zookeeperUtils;
        }

        @Override
        protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) {
            createdTopics.put(topicName, true);
        }

        @Override
        protected boolean handleSecurity(Configuration atlasProperties) {
            return false;
        }
    };
    atlasTopicCreator.createAtlasTopic(configuration, "ATLAS_HOOK", "ATLAS_ENTITIES");
    assertFalse(createdTopics.get("ATLAS_HOOK"));
    assertFalse(createdTopics.get("ATLAS_ENTITIES"));
}
Also used : Configuration(org.apache.commons.configuration.Configuration) HashMap(java.util.HashMap) ZkUtils(kafka.utils.ZkUtils) Test(org.testng.annotations.Test)

Example 24 with ZkUtils

use of kafka.utils.ZkUtils in project atlas by apache.

the class AtlasTopicCreatorTest method shouldNotCreateTopicIfItAlreadyExists.

@Test
public void shouldNotCreateTopicIfItAlreadyExists() {
    Configuration configuration = mock(Configuration.class);
    when(configuration.getBoolean(AtlasTopicCreator.ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)).thenReturn(true);
    when(configuration.getString("atlas.authentication.method.kerberos")).thenReturn("false");
    final ZkUtils zookeeperUtils = mock(ZkUtils.class);
    final boolean[] topicExistsCalled = new boolean[] { false };
    final boolean[] createTopicCalled = new boolean[] { false };
    AtlasTopicCreator atlasTopicCreator = new AtlasTopicCreator() {

        @Override
        protected boolean ifTopicExists(String topicName, ZkUtils zkUtils) {
            topicExistsCalled[0] = true;
            return true;
        }

        @Override
        protected ZkUtils createZkUtils(Configuration atlasProperties) {
            return zookeeperUtils;
        }

        @Override
        protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) {
            createTopicCalled[0] = true;
        }
    };
    atlasTopicCreator.createAtlasTopic(configuration, "ATLAS_HOOK");
    assertTrue(topicExistsCalled[0]);
    assertFalse(createTopicCalled[0]);
}
Also used : Configuration(org.apache.commons.configuration.Configuration) ZkUtils(kafka.utils.ZkUtils) Test(org.testng.annotations.Test)

Example 25 with ZkUtils

use of kafka.utils.ZkUtils in project atlas by apache.

the class AtlasTopicCreator method createAtlasTopic.

/**
 * Create an Atlas topic.
 *
 * The topic will get created based on following conditions:
 * {@link #ATLAS_NOTIFICATION_CREATE_TOPICS_KEY} is set to true.
 * The topic does not already exist.
 * Note that despite this, there could be multiple topic creation calls that happen in parallel because hooks
 * run in a distributed fashion. Exceptions are caught and logged by this method to prevent the startup of
 * the hooks from failing.
 * @param atlasProperties {@link Configuration} containing properties to be used for creating topics.
 * @param topicNames list of topics to create
 */
public void createAtlasTopic(Configuration atlasProperties, String... topicNames) {
    if (atlasProperties.getBoolean(ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)) {
        if (!handleSecurity(atlasProperties)) {
            return;
        }
        ZkUtils zkUtils = createZkUtils(atlasProperties);
        for (String topicName : topicNames) {
            try {
                LOG.warn("Attempting to create topic {}", topicName);
                if (!ifTopicExists(topicName, zkUtils)) {
                    createTopic(atlasProperties, topicName, zkUtils);
                } else {
                    LOG.warn("Ignoring call to create topic {}, as it already exists.", topicName);
                }
            } catch (Throwable t) {
                LOG.error("Failed while creating topic {}", topicName, t);
            }
        }
        zkUtils.close();
    } else {
        LOG.info("Not creating topics {} as {} is false", StringUtils.join(topicNames, ","), ATLAS_NOTIFICATION_CREATE_TOPICS_KEY);
    }
}
Also used : ZkUtils(kafka.utils.ZkUtils)

Aggregations

ZkUtils (kafka.utils.ZkUtils)61 ZkClient (org.I0Itec.zkclient.ZkClient)26 Properties (java.util.Properties)25 ZkConnection (org.I0Itec.zkclient.ZkConnection)22 Test (org.testng.annotations.Test)18 Configuration (org.apache.commons.configuration.Configuration)16 HashMap (java.util.HashMap)8 KafkaConfig (kafka.server.KafkaConfig)8 TestingServer (org.apache.curator.test.TestingServer)8 KafkaServerStartable (kafka.server.KafkaServerStartable)7 ServerSocket (java.net.ServerSocket)6 Test (org.junit.Test)4 File (java.io.File)3 Path (java.nio.file.Path)2 Level (java.util.logging.Level)2 TopicMetadata (kafka.api.TopicMetadata)2 TopicExistsException (kafka.common.TopicExistsException)2 InstanceSpec (org.apache.curator.test.InstanceSpec)2 TopicDescription (org.apache.kafka.clients.admin.TopicDescription)2 Before (org.junit.Before)2