Examples with KafkaException org.apache.kafka.common.KafkaException used on opensource projects

Example 96 with KafkaException

use of org.apache.kafka.common.KafkaException in project kafka by apache.

the class ClientCompatibilityTest method testDescribeConfigsMethod.

private void testDescribeConfigsMethod(final Admin client) throws Throwable {
    tryFeature("describeConfigsSupported", testConfig.describeConfigsSupported, () -> {
        try {
            Collection<Node> nodes = client.describeCluster().nodes().get();
            final ConfigResource configResource = new ConfigResource(ConfigResource.Type.BROKER, nodes.iterator().next().idString());
            Map<ConfigResource, Config> brokerConfig = client.describeConfigs(Collections.singleton(configResource)).all().get();
            if (brokerConfig.get(configResource).entries().isEmpty()) {
                throw new KafkaException("Expected to see config entries, but got zero entries");
            }
        } catch (ExecutionException e) {
            throw e.getCause();
        }
    });
}

Example 97 with KafkaException

use of org.apache.kafka.common.KafkaException in project kafka by apache.

the class TopicBasedRemoteLogMetadataManagerHarness method initializeRemoteLogMetadataManager.

public void initializeRemoteLogMetadataManager(Set<TopicIdPartition> topicIdPartitions, boolean startConsumerThread) {
    String logDir = TestUtils.tempDirectory("rlmm_segs_").getAbsolutePath();
    topicBasedRemoteLogMetadataManager = new TopicBasedRemoteLogMetadataManager(startConsumerThread) {

        @Override
        public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions, Set<TopicIdPartition> followerPartitions) {
            Set<TopicIdPartition> allReplicas = new HashSet<>(leaderPartitions);
            allReplicas.addAll(followerPartitions);
            // Make sure the topic partition dirs exist as the topics might not have been created on this broker.
            for (TopicIdPartition topicIdPartition : allReplicas) {
                // Create partition directory in the log directory created by topicBasedRemoteLogMetadataManager.
                File partitionDir = new File(new File(config().logDir()), topicIdPartition.topicPartition().topic() + "-" + topicIdPartition.topicPartition().partition());
                partitionDir.mkdirs();
                if (!partitionDir.exists()) {
                    throw new KafkaException("Partition directory:[" + partitionDir + "] could not be created successfully.");
                }
            }
            super.onPartitionLeadershipChanges(leaderPartitions, followerPartitions);
        }
    };
    // Initialize TopicBasedRemoteLogMetadataManager.
    Map<String, Object> configs = new HashMap<>();
    configs.put(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX + CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(listenerName()));
    configs.put(BROKER_ID, 0);
    configs.put(LOG_DIR, logDir);
    configs.put(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, METADATA_TOPIC_PARTITIONS_COUNT);
    configs.put(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, METADATA_TOPIC_REPLICATION_FACTOR);
    configs.put(REMOTE_LOG_METADATA_TOPIC_RETENTION_MS_PROP, METADATA_TOPIC_RETENTION_MS);
    log.debug("TopicBasedRemoteLogMetadataManager configs before adding overridden properties: {}", configs);
    // Add override properties.
    configs.putAll(overrideRemoteLogMetadataManagerProps());
    log.debug("TopicBasedRemoteLogMetadataManager configs after adding overridden properties: {}", configs);
    topicBasedRemoteLogMetadataManager.configure(configs);
    try {
        waitUntilInitialized(60_000);
    } catch (TimeoutException e) {
        throw new KafkaException(e);
    }
    topicBasedRemoteLogMetadataManager.onPartitionLeadershipChanges(topicIdPartitions, Collections.emptySet());
}

Example 98 with KafkaException

use of org.apache.kafka.common.KafkaException in project kafka by apache.

the class ConsumerTask method maybeWaitForPartitionsAssignment.

private void maybeWaitForPartitionsAssignment() {
    Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
    synchronized (assignPartitionsLock) {
        // in close() method with in the same lock to avoid any race conditions.
        if (closing) {
            return;
        }
        while (assignedMetaPartitions.isEmpty()) {
            // If no partitions are assigned, wait until they are assigned.
            log.debug("Waiting for assigned remote log metadata partitions..");
            try {
                // No timeout is set here, as it is always notified. Even when it is closed, the race can happen
                // between the thread calling this method and the thread calling close(). We should have a check
                // for closing as that might have been set and notified with assignPartitionsLock by `close`
                // method.
                assignPartitionsLock.wait();
                if (closing) {
                    return;
                }
            } catch (InterruptedException e) {
                throw new KafkaException(e);
            }
        }
        if (assignPartitions) {
            assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
            // Removing unassigned meta partitions from partitionToConsumedOffsets and partitionToCommittedOffsets
            partitionToConsumedOffsets.entrySet().removeIf(entry -> !assignedMetaPartitions.contains(entry.getKey()));
            assignPartitions = false;
        }
    }
    if (!assignedMetaPartitionsSnapshot.isEmpty()) {
        executeReassignment(assignedMetaPartitionsSnapshot);
    }
}

Example 99 with KafkaException

use of org.apache.kafka.common.KafkaException in project kafka by apache.

the class ConsumerManager method startConsumerThread.

public void startConsumerThread() {
    try {
        // Start a thread to continuously consume records from topic partitions.
        consumerTaskThread.start();
        log.info("RLMM Consumer task thread is started");
    } catch (Exception e) {
        throw new KafkaException("Error encountered while initializing and scheduling ConsumerTask thread", e);
    }
}

Example 100 with KafkaException

use of org.apache.kafka.common.KafkaException in project kafka by apache.

the class ConsumerTask method maybeSyncCommittedDataAndOffsets.

private void maybeSyncCommittedDataAndOffsets(boolean forceSync) {
    // Return immediately if there is no consumption from last time.
    boolean noConsumedOffsetUpdates = partitionToConsumedOffsets.equals(lastSyncedPartitionToConsumedOffsets);
    if (noConsumedOffsetUpdates || !forceSync && time.milliseconds() - lastSyncedTimeMs < committedOffsetSyncIntervalMs) {
        log.debug("Skip syncing committed offsets, noConsumedOffsetUpdates: {}, forceSync: {}", noConsumedOffsetUpdates, forceSync);
        return;
    }
    try {
        // get updated by other threads.
        synchronized (assignPartitionsLock) {
            for (TopicIdPartition topicIdPartition : assignedTopicPartitions) {
                int metadataPartition = topicPartitioner.metadataPartition(topicIdPartition);
                Long offset = partitionToConsumedOffsets.get(metadataPartition);
                if (offset != null) {
                    remotePartitionMetadataEventHandler.syncLogMetadataSnapshot(topicIdPartition, metadataPartition, offset);
                } else {
                    log.debug("Skipping syncup of the remote-log-metadata-file for partition:{} , with remote log metadata partition{}, and no offset", topicIdPartition, metadataPartition);
                }
            }
            // Write partitionToConsumedOffsets into committed offsets file as we do not want to process them again
            // in case of restarts.
            committedOffsetsFile.writeEntries(partitionToConsumedOffsets);
            lastSyncedPartitionToConsumedOffsets = new HashMap<>(partitionToConsumedOffsets);
        }
        lastSyncedTimeMs = time.milliseconds();
    } catch (IOException e) {
        throw new KafkaException("Error encountered while writing committed offsets to a local file", e);
    }
}