Examples with PartitionInfo org.apache.kafka.common.PartitionInfo used on opensource projects

Example 61 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project apache-kafka-on-k8s by banzaicloud.

the class MockConsumer method subscribe.

@Override
public synchronized void subscribe(Pattern pattern, final ConsumerRebalanceListener listener) {
    ensureNotClosed();
    committed.clear();
    this.subscriptions.subscribe(pattern, listener);
    Set<String> topicsToSubscribe = new HashSet<>();
    for (String topic : partitions.keySet()) {
        if (pattern.matcher(topic).matches() && !subscriptions.subscription().contains(topic))
            topicsToSubscribe.add(topic);
    }
    ensureNotClosed();
    this.subscriptions.subscribeFromPattern(topicsToSubscribe);
    final Set<TopicPartition> assignedPartitions = new HashSet<>();
    for (final String topic : topicsToSubscribe) {
        for (final PartitionInfo info : this.partitions.get(topic)) {
            assignedPartitions.add(new TopicPartition(topic, info.partition()));
        }
    }
    subscriptions.assignFromSubscribed(assignedPartitions);
}

Example 62 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project apache-kafka-on-k8s by banzaicloud.

the class KafkaConsumer method partitionsFor.

/**
 * Get metadata about the partitions for a given topic. This method will issue a remote call to the server if it
 * does not already have any metadata about the given topic.
 *
 * @param topic The topic to get partition metadata for
 * @return The list of partitions
 * @throws org.apache.kafka.common.errors.WakeupException if {@link #wakeup()} is called before or while this
 *             function is called
 * @throws org.apache.kafka.common.errors.InterruptException if the calling thread is interrupted before or while
 *             this function is called
 * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details
 * @throws org.apache.kafka.common.errors.AuthorizationException if not authorized to the specified topic. See the exception for more details
 * @throws org.apache.kafka.common.errors.TimeoutException if the topic metadata could not be fetched before
 *             expiration of the configured request timeout
 * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors
 */
@Override
public List<PartitionInfo> partitionsFor(String topic) {
    acquireAndEnsureOpen();
    try {
        Cluster cluster = this.metadata.fetch();
        List<PartitionInfo> parts = cluster.partitionsForTopic(topic);
        if (!parts.isEmpty())
            return parts;
        Map<String, List<PartitionInfo>> topicMetadata = fetcher.getTopicMetadata(new MetadataRequest.Builder(Collections.singletonList(topic), true), requestTimeoutMs);
        return topicMetadata.get(topic);
    } finally {
        release();
    }
}

Example 63 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project drill by apache.

the class KafkaMessageGenerator method populateJsonMsgWithTimestamps.

public void populateJsonMsgWithTimestamps(String topic, int numMsg) throws ExecutionException, InterruptedException {
    try (KafkaProducer<String, String> producer = new KafkaProducer<>(producerProperties)) {
        int halfCount = numMsg / 2;
        for (PartitionInfo tpInfo : producer.partitionsFor(topic)) {
            for (int i = 1; i <= numMsg; ++i) {
                JsonObject object = new JsonObject();
                object.addProperty("stringKey", UUID.randomUUID().toString());
                object.addProperty("intKey", numMsg - i);
                object.addProperty("boolKey", i % 2 == 0);
                long timestamp = i < halfCount ? (halfCount - i) : i;
                ProducerRecord<String, String> message = new ProducerRecord<>(tpInfo.topic(), tpInfo.partition(), timestamp, "key" + i, object.toString());
                logger.info("Publishing message : {}", message);
                Future<RecordMetadata> future = producer.send(message);
                logger.info("Committed offset of the message : {}", future.get().offset());
            }
        }
    }
}

Example 64 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project samza by apache.

the class KafkaSystemAdmin method getSystemStreamPartitionCounts.

/**
 * Note! This method does not populate SystemStreamMetadata for each stream with real data.
 * Thus, this method should ONLY be used to get number of partitions for each stream.
 * It will throw NotImplementedException if anyone tries to access the actual metadata.
 * @param streamNames set of streams for which get the partitions counts
 * @param cacheTTL cache TTL if caching the data
 * @return a map, keyed on stream names. Number of partitions in SystemStreamMetadata is the output of this method.
 */
@Override
public Map<String, SystemStreamMetadata> getSystemStreamPartitionCounts(Set<String> streamNames, long cacheTTL) {
    // This optimization omits actual metadata for performance. Instead, we inject a dummy for all partitions.
    final SystemStreamMetadata.SystemStreamPartitionMetadata dummySspm = new SystemStreamMetadata.SystemStreamPartitionMetadata(null, null, null) {

        String msg = "getSystemStreamPartitionCounts does not populate SystemStreaMetadata info. Only number of partitions";

        @Override
        public String getOldestOffset() {
            throw new NotImplementedException(msg);
        }

        @Override
        public String getNewestOffset() {
            throw new NotImplementedException(msg);
        }

        @Override
        public String getUpcomingOffset() {
            throw new NotImplementedException(msg);
        }
    };
    ExponentialSleepStrategy strategy = new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS);
    Function1<ExponentialSleepStrategy.RetryLoop, Map<String, SystemStreamMetadata>> fetchMetadataOperation = new AbstractFunction1<ExponentialSleepStrategy.RetryLoop, Map<String, SystemStreamMetadata>>() {

        @Override
        public Map<String, SystemStreamMetadata> apply(ExponentialSleepStrategy.RetryLoop loop) {
            Map<String, SystemStreamMetadata> allMetadata = new HashMap<>();
            streamNames.forEach(streamName -> {
                Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>();
                List<PartitionInfo> partitionInfos = threadSafeKafkaConsumer.execute(consumer -> consumer.partitionsFor(streamName));
                LOG.debug("Stream {} has partitions {}", streamName, partitionInfos);
                partitionInfos.forEach(partitionInfo -> partitionMetadata.put(new Partition(partitionInfo.partition()), dummySspm));
                allMetadata.put(streamName, new SystemStreamMetadata(streamName, partitionMetadata));
            });
            loop.done();
            return allMetadata;
        }
    };
    Map<String, SystemStreamMetadata> result = strategy.run(fetchMetadataOperation, new AbstractFunction2<Exception, ExponentialSleepStrategy.RetryLoop, BoxedUnit>() {

        @Override
        public BoxedUnit apply(Exception exception, ExponentialSleepStrategy.RetryLoop loop) {
            if (loop.sleepCount() < MAX_RETRIES_ON_EXCEPTION) {
                LOG.warn(String.format("Fetching systemstreampartition counts for: %s threw an exception. Retrying.", streamNames), exception);
            } else {
                LOG.error(String.format("Fetching systemstreampartition counts for: %s threw an exception.", streamNames), exception);
                loop.done();
                throw new SamzaException(exception);
            }
            return null;
        }
    }).get();
    LOG.info("SystemStream partition counts for system {}: {}", systemName, result);
    return result;
}

Example 65 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project samza by apache.

the class TestKafkaSystemAdminWithMock method testGetSystemStreamMetaDataWithRetry.

@Test
public void testGetSystemStreamMetaDataWithRetry() {
    final List<PartitionInfo> partitionInfosForTopic = ImmutableList.of(mockPartitionInfo0, mockPartitionInfo1);
    when(mockKafkaConsumer.partitionsFor(VALID_TOPIC)).thenThrow(new RuntimeException()).thenReturn(partitionInfosForTopic);
    Map<String, SystemStreamMetadata> metadataMap = kafkaSystemAdmin.getSystemStreamMetadata(ImmutableSet.of(VALID_TOPIC));
    assertEquals("metadata should return for 1 topic", metadataMap.size(), 1);
    // retried twice because the first fails and the second succeeds
    Mockito.verify(mockKafkaConsumer, Mockito.times(2)).partitionsFor(VALID_TOPIC);
    final List<TopicPartition> topicPartitions = Arrays.asList(new TopicPartition(mockPartitionInfo0.topic(), mockPartitionInfo0.partition()), new TopicPartition(mockPartitionInfo1.topic(), mockPartitionInfo1.partition()));
    // the following methods thereafter are only called once
    Mockito.verify(mockKafkaConsumer, Mockito.times(1)).beginningOffsets(topicPartitions);
    Mockito.verify(mockKafkaConsumer, Mockito.times(1)).endOffsets(topicPartitions);
}