Examples with TopicPartition org.apache.kafka.common.TopicPartition used on opensource projects

Example 26 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class AssignmentInfo method encode.

/**
     * @throws TaskAssignmentException if method fails to encode the data, e.g., if there is an
     * IO exception during encoding
     */
public ByteBuffer encode() {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    try {
        // Encode version
        out.writeInt(version);
        // Encode active tasks
        out.writeInt(activeTasks.size());
        for (TaskId id : activeTasks) {
            id.writeTo(out);
        }
        // Encode standby tasks
        out.writeInt(standbyTasks.size());
        for (Map.Entry<TaskId, Set<TopicPartition>> entry : standbyTasks.entrySet()) {
            TaskId id = entry.getKey();
            id.writeTo(out);
            Set<TopicPartition> partitions = entry.getValue();
            writeTopicPartitions(out, partitions);
        }
        out.writeInt(partitionsByHost.size());
        for (Map.Entry<HostInfo, Set<TopicPartition>> entry : partitionsByHost.entrySet()) {
            final HostInfo hostInfo = entry.getKey();
            out.writeUTF(hostInfo.host());
            out.writeInt(hostInfo.port());
            writeTopicPartitions(out, entry.getValue());
        }
        out.flush();
        out.close();
        return ByteBuffer.wrap(baos.toByteArray());
    } catch (IOException ex) {
        throw new TaskAssignmentException("Failed to encode AssignmentInfo", ex);
    }
}

Example 27 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class AssignmentInfo method writeTopicPartitions.

private void writeTopicPartitions(DataOutputStream out, Set<TopicPartition> partitions) throws IOException {
    out.writeInt(partitions.size());
    for (TopicPartition partition : partitions) {
        out.writeUTF(partition.topic());
        out.writeInt(partition.partition());
    }
}

Example 28 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class StreamsResetter method resetInputAndInternalAndSeekToEndIntermediateTopicOffsets.

private void resetInputAndInternalAndSeekToEndIntermediateTopicOffsets() {
    final List<String> inputTopics = options.valuesOf(inputTopicsOption);
    final List<String> intermediateTopics = options.valuesOf(intermediateTopicsOption);
    if (inputTopics.size() == 0 && intermediateTopics.size() == 0) {
        System.out.println("No input or intermediate topics specified. Skipping seek.");
        return;
    } else {
        if (inputTopics.size() != 0) {
            System.out.println("Resetting offsets to zero for input topics " + inputTopics + " and all internal topics.");
        }
        if (intermediateTopics.size() != 0) {
            System.out.println("Seek-to-end for intermediate topics " + intermediateTopics);
        }
    }
    final Properties config = new Properties();
    config.putAll(consumerConfig);
    config.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, options.valueOf(bootstrapServerOption));
    config.setProperty(ConsumerConfig.GROUP_ID_CONFIG, options.valueOf(applicationIdOption));
    config.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
    final Set<String> topicsToSubscribe = new HashSet<>(inputTopics.size() + intermediateTopics.size());
    for (final String topic : inputTopics) {
        if (!allTopics.contains(topic)) {
            System.err.println("Input topic " + topic + " not found. Skipping.");
        } else {
            topicsToSubscribe.add(topic);
        }
    }
    for (final String topic : intermediateTopics) {
        if (!allTopics.contains(topic)) {
            System.err.println("Intermediate topic " + topic + " not found. Skipping.");
        } else {
            topicsToSubscribe.add(topic);
        }
    }
    for (final String topic : allTopics) {
        if (isInternalTopic(topic)) {
            topicsToSubscribe.add(topic);
        }
    }
    try (final KafkaConsumer<byte[], byte[]> client = new KafkaConsumer<>(config, new ByteArrayDeserializer(), new ByteArrayDeserializer())) {
        client.subscribe(topicsToSubscribe);
        client.poll(1);
        final Set<TopicPartition> partitions = client.assignment();
        final Set<TopicPartition> inputAndInternalTopicPartitions = new HashSet<>();
        final Set<TopicPartition> intermediateTopicPartitions = new HashSet<>();
        for (final TopicPartition p : partitions) {
            final String topic = p.topic();
            if (isInputTopic(topic) || isInternalTopic(topic)) {
                inputAndInternalTopicPartitions.add(p);
            } else if (isIntermediateTopic(topic)) {
                intermediateTopicPartitions.add(p);
            } else {
                System.err.println("Skipping invalid partition: " + p);
            }
        }
        if (inputAndInternalTopicPartitions.size() > 0) {
            client.seekToBeginning(inputAndInternalTopicPartitions);
        }
        if (intermediateTopicPartitions.size() > 0) {
            client.seekToEnd(intermediateTopicPartitions);
        }
        for (final TopicPartition p : partitions) {
            client.position(p);
        }
        client.commitSync();
    } catch (final RuntimeException e) {
        System.err.println("ERROR: Resetting offsets failed.");
        throw e;
    }
    System.out.println("Done.");
}

Example 29 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class StreamPartitionAssignor method onAssignment.

/**
     * @throws TaskAssignmentException if there is no task id for one of the partitions specified
     */
@Override
public void onAssignment(Assignment assignment) {
    List<TopicPartition> partitions = new ArrayList<>(assignment.partitions());
    Collections.sort(partitions, PARTITION_COMPARATOR);
    AssignmentInfo info = AssignmentInfo.decode(assignment.userData());
    this.standbyTasks = info.standbyTasks;
    this.activeTasks = new HashMap<>();
    // could be duplicated if one task has more than one assigned partitions
    if (partitions.size() != info.activeTasks.size()) {
        throw new TaskAssignmentException(String.format("stream-thread [%s] Number of assigned partitions %d is not equal to the number of active taskIds %d" + ", assignmentInfo=%s", streamThread.getName(), partitions.size(), info.activeTasks.size(), info.toString()));
    }
    for (int i = 0; i < partitions.size(); i++) {
        TopicPartition partition = partitions.get(i);
        TaskId id = info.activeTasks.get(i);
        Set<TopicPartition> assignedPartitions = activeTasks.get(id);
        if (assignedPartitions == null) {
            assignedPartitions = new HashSet<>();
            activeTasks.put(id, assignedPartitions);
        }
        assignedPartitions.add(partition);
    }
    this.partitionsByHostState = info.partitionsByHost;
    final Collection<Set<TopicPartition>> values = partitionsByHostState.values();
    final Map<TopicPartition, PartitionInfo> topicToPartitionInfo = new HashMap<>();
    for (Set<TopicPartition> value : values) {
        for (TopicPartition topicPartition : value) {
            topicToPartitionInfo.put(topicPartition, new PartitionInfo(topicPartition.topic(), topicPartition.partition(), null, new Node[0], new Node[0]));
        }
    }
    metadataWithInternalTopics = Cluster.empty().withPartitions(topicToPartitionInfo);
}

Example 30 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class StreamThread method runLoop.

private void runLoop() {
    int totalNumBuffered = 0;
    boolean requiresPoll = true;
    boolean polledRecords = false;
    consumer.subscribe(sourceTopicPattern, rebalanceListener);
    while (stillRunning()) {
        this.timerStartedMs = time.milliseconds();
        // try to fetch some records if necessary
        if (requiresPoll) {
            requiresPoll = false;
            boolean longPoll = totalNumBuffered == 0;
            ConsumerRecords<byte[], byte[]> records = null;
            try {
                records = consumer.poll(longPoll ? this.pollTimeMs : 0);
            } catch (NoOffsetForPartitionException ex) {
                TopicPartition partition = ex.partition();
                if (builder.earliestResetTopicsPattern().matcher(partition.topic()).matches()) {
                    log.info(String.format("stream-thread [%s] setting topic to consume from earliest offset %s", this.getName(), partition.topic()));
                    consumer.seekToBeginning(ex.partitions());
                } else if (builder.latestResetTopicsPattern().matcher(partition.topic()).matches()) {
                    consumer.seekToEnd(ex.partitions());
                    log.info(String.format("stream-thread [%s] setting topic to consume from latest offset %s", this.getName(), partition.topic()));
                } else {
                    if (originalReset == null || (!originalReset.equals("earliest") && !originalReset.equals("latest"))) {
                        setState(State.PENDING_SHUTDOWN);
                        String errorMessage = "No valid committed offset found for input topic %s (partition %s) and no valid reset policy configured." + " You need to set configuration parameter \"auto.offset.reset\" or specify a topic specific reset " + "policy via KStreamBuilder#stream(StreamsConfig.AutoOffsetReset offsetReset, ...) or KStreamBuilder#table(StreamsConfig.AutoOffsetReset offsetReset, ...)";
                        throw new StreamsException(String.format(errorMessage, partition.topic(), partition.partition()), ex);
                    }
                    if (originalReset.equals("earliest")) {
                        consumer.seekToBeginning(ex.partitions());
                    } else if (originalReset.equals("latest")) {
                        consumer.seekToEnd(ex.partitions());
                    }
                    log.info(String.format("stream-thread [%s] no custom setting defined for topic %s using original config %s for offset reset", this.getName(), partition.topic(), originalReset));
                }
            }
            if (rebalanceException != null)
                throw new StreamsException(logPrefix + " Failed to rebalance", rebalanceException);
            if (records != null && !records.isEmpty()) {
                int numAddedRecords = 0;
                for (TopicPartition partition : records.partitions()) {
                    StreamTask task = activeTasksByPartition.get(partition);
                    numAddedRecords += task.addRecords(partition, records.records(partition));
                }
                streamsMetrics.skippedRecordsSensor.record(records.count() - numAddedRecords, timerStartedMs);
                polledRecords = true;
            } else {
                polledRecords = false;
            }
            // only record poll latency is long poll is required
            if (longPoll) {
                streamsMetrics.pollTimeSensor.record(computeLatency(), timerStartedMs);
            }
        }
        // functions if necessary, which may result in more records going through the topology in this loop
        if (totalNumBuffered > 0 || polledRecords) {
            totalNumBuffered = 0;
            if (!activeTasks.isEmpty()) {
                for (StreamTask task : activeTasks.values()) {
                    totalNumBuffered += task.process();
                    requiresPoll = requiresPoll || task.requiresPoll();
                    streamsMetrics.processTimeSensor.record(computeLatency(), timerStartedMs);
                    maybePunctuate(task);
                    if (task.commitNeeded())
                        commitOne(task);
                }
            } else {
                // even when no task is assigned, we must poll to get a task.
                requiresPoll = true;
            }
        } else {
            requiresPoll = true;
        }
        maybeCommit(timerStartedMs);
        maybeUpdateStandbyTasks();
        maybeClean(timerStartedMs);
    }
    log.info("{} Shutting down at user request", logPrefix);
}