Examples with TopicPartition org.apache.kafka.common.TopicPartition used on opensource projects

Example 76 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project streamsx.kafka by IBMStreams.

the class KafkaConsumerClient method seekToTimestamp.

private void seekToTimestamp(Map<TopicPartition, Long> topicPartitionTimestampMap) {
    Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes = consumer.offsetsForTimes(topicPartitionTimestampMap);
    logger.debug("offsetsForTimes=" + offsetsForTimes);
    topicPartitionTimestampMap.forEach((tp, timestamp) -> {
        OffsetAndTimestamp ot = offsetsForTimes.get(tp);
        if (ot != null) {
            logger.debug("Seeking consumer for tp=" + tp + " to offsetAndTimestamp=" + ot);
            consumer.seek(tp, ot.offset());
        } else {
        // nothing...consumer will move to the offset as determined by the 'auto.offset.reset' config
        }
    });
}

Example 77 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project streamsx.kafka by IBMStreams.

the class TransactionalKafkaProducerClient method checkpoint.

@Override
public void checkpoint(Checkpoint checkpoint) throws Exception {
    final long currentSequenceId = checkpoint.getSequenceId();
    if (logger.isDebugEnabled())
        logger.debug("TransactionalKafkaProducerClient -- CHECKPOINT id=" + currentSequenceId);
    // when we checkpoint, we must have a transaction. open a transaction if not yet done ...
    checkAndBeginTransaction();
    if (logger.isDebugEnabled())
        logger.debug("currentSequenceId=" + currentSequenceId + ", lastSuccessSequenceId=" + lastSuccessfulSequenceId);
    boolean doCommit = true;
    // that's why the second condition is checked first.
    if (currentSequenceId > lastSuccessfulSequenceId + 1) {
        // must be read with 'isolation.level=read_committed'
        long committedSequenceId = getCommittedSequenceIdFromCtrlTopic();
        if (logger.isDebugEnabled())
            logger.debug("committedSequenceId=" + committedSequenceId);
        if (lastSuccessfulSequenceId < committedSequenceId) {
            if (logger.isDebugEnabled())
                logger.debug("Aborting transaction due to lastSuccessfulSequenceId < committedSequenceId");
            // If the last successful sequence ID is less than
            // the committed sequence ID, this transaction has
            // been processed before and is a duplicate.
            // Discard this transaction.
            abortTransaction();
            doCommit = false;
            lastSuccessfulSequenceId = committedSequenceId;
        }
    }
    if (logger.isDebugEnabled())
        logger.debug("doCommit = " + doCommit);
    if (doCommit) {
        RecordMetadata lastCommittedControlRecordMetadata = commitTransaction(currentSequenceId);
        lastSuccessfulSequenceId = currentSequenceId;
        TopicPartition tp = new TopicPartition(lastCommittedControlRecordMetadata.topic(), lastCommittedControlRecordMetadata.partition());
        controlTopicInitialOffsets.put(tp, lastCommittedControlRecordMetadata.offset());
    // The 'controlTopicInitialOffsets' need not be synced back to the JCP. The CV is for reset to initial state.
    // this.startOffsetsCV.setValue (serializeObject (controlTopicInitialOffsets));
    }
    transactionInProgress.set(false);
    // save the last successful seq ID
    if (logger.isDebugEnabled())
        logger.debug("Checkpointing lastSuccessfulSequenceId: " + lastSuccessfulSequenceId);
    checkpoint.getOutputStream().writeLong(lastSuccessfulSequenceId);
    // save the control topic offsets
    if (logger.isDebugEnabled())
        logger.debug("Checkpointing control topic offsets: " + controlTopicInitialOffsets);
    checkpoint.getOutputStream().writeObject(controlTopicInitialOffsets);
    if (!lazyTransactionBegin) {
        // start a new transaction
        checkAndBeginTransaction();
    }
}

Example 78 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project streamsx.kafka by IBMStreams.

the class AbstractKafkaConsumerOperator method process.

@Override
public void process(StreamingInput<Tuple> stream, Tuple tuple) throws Exception {
    boolean interrupted = false;
    try {
        String jsonString = tuple.getString(0);
        JsonObject jsonObj = gson.fromJson(jsonString, JsonObject.class);
        TopicPartitionUpdateAction action = null;
        if (jsonObj.has("action")) {
            // $NON-NLS-1$
            // $NON-NLS-1$
            action = TopicPartitionUpdateAction.valueOf(jsonObj.get("action").getAsString().toUpperCase());
        } else {
            // $NON-NLS-1$ //$NON-NLS-2$
            logger.error(Messages.getString("INVALID_JSON_MISSING_KEY", "action", jsonString));
            return;
        }
        Map<TopicPartition, Long> topicPartitionOffsetMap = null;
        if (jsonObj.has("topicPartitionOffsets")) {
            // $NON-NLS-1$
            topicPartitionOffsetMap = new HashMap<TopicPartition, Long>();
            // $NON-NLS-1$
            JsonArray arr = jsonObj.get("topicPartitionOffsets").getAsJsonArray();
            Iterator<JsonElement> it = arr.iterator();
            while (it.hasNext()) {
                JsonObject tpo = it.next().getAsJsonObject();
                if (!tpo.has("topic")) {
                    // $NON-NLS-1$
                    // $NON-NLS-1$ //$NON-NLS-2$
                    logger.error(Messages.getString("INVALID_JSON_MISSING_KEY", "topic", jsonString));
                    return;
                }
                if (!tpo.has("partition")) {
                    // $NON-NLS-1$
                    // $NON-NLS-1$ //$NON-NLS-2$
                    logger.error(Messages.getString("INVALID_JSON_MISSING_KEY", "partition", jsonString));
                    return;
                }
                if (action == TopicPartitionUpdateAction.ADD && !tpo.has("offset")) {
                    // $NON-NLS-1$
                    // $NON-NLS-1$ //$NON-NLS-2$
                    logger.error(Messages.getString("INVALID_JSON_MISSING_KEY", "offset", jsonString));
                    return;
                }
                // $NON-NLS-1$
                String topic = tpo.get("topic").getAsString();
                // $NON-NLS-1$
                int partition = tpo.get("partition").getAsInt();
                // $NON-NLS-1$ //$NON-NLS-2$
                long offset = tpo.has("offset") ? tpo.get("offset").getAsLong() : 0l;
                topicPartitionOffsetMap.put(new TopicPartition(topic, partition), offset);
            }
        }
        consumer.sendStopPollingEvent();
        consumer.sendUpdateTopicAssignmentEvent(new TopicPartitionUpdate(action, topicPartitionOffsetMap));
    } catch (InterruptedException e) {
        // interrupted during shutdown
        interrupted = true;
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
    } finally {
        if (!interrupted)
            consumer.sendStartPollingEvent(consumerPollTimeout);
    }
}

Example 79 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project apex-malhar by apache.

the class KafkaSinglePortExactlyOnceOutputOperator method rebuildPartialWindow.

private void rebuildPartialWindow() {
    logger.info("Rebuild the partial window after " + windowDataManager.getLargestCompletedWindow());
    Map<Integer, Long> storedOffsets;
    Map<Integer, Long> currentOffsets;
    try {
        storedOffsets = (Map<Integer, Long>) this.windowDataManager.retrieve(windowId);
        currentOffsets = getPartitionsAndOffsets(true);
    } catch (IOException | ExecutionException | InterruptedException e) {
        throw new RuntimeException(e);
    }
    if (currentOffsets == null) {
        logger.info("No tuples found while building partial window " + windowDataManager.getLargestCompletedWindow());
        return;
    }
    if (storedOffsets == null) {
        logger.info("Stored offset not available, seeking to the beginning of the Kafka Partition.");
        try {
            storedOffsets = getPartitionsAndOffsets(false);
        } catch (ExecutionException | InterruptedException e) {
            throw new RuntimeException(e);
        }
    }
    List<TopicPartition> topicPartitions = new ArrayList<>();
    for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
        topicPartitions.add(new TopicPartition(getTopic(), entry.getKey()));
    }
    consumer.assign(topicPartitions);
    for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
        Long storedOffset = 0L;
        Integer currentPartition = entry.getKey();
        Long currentOffset = entry.getValue();
        if (storedOffsets.containsKey(currentPartition)) {
            storedOffset = storedOffsets.get(currentPartition);
        }
        if (storedOffset >= currentOffset) {
            continue;
        }
        try {
            consumer.seek(new TopicPartition(getTopic(), currentPartition), storedOffset);
        } catch (Exception ex) {
            logger.info("Rebuilding of the partial window is not complete, exactly once recovery is not possible.");
            throw new RuntimeException(ex);
        }
        int kafkaAttempt = 0;
        while (true) {
            ConsumerRecords<String, T> consumerRecords = consumer.poll(100);
            if (consumerRecords.count() == 0) {
                if (kafkaAttempt++ == KAFKA_CONNECT_ATTEMPT) {
                    break;
                }
            } else {
                kafkaAttempt = 0;
            }
            boolean crossedBoundary = false;
            for (ConsumerRecord<String, T> consumerRecord : consumerRecords) {
                if (consumerRecord.offset() >= currentOffset) {
                    crossedBoundary = true;
                    break;
                }
                if (!doesKeyBelongsToThisInstance(operatorId, consumerRecord.key())) {
                    continue;
                }
                T value = consumerRecord.value();
                if (partialWindowTuples.containsKey(value)) {
                    Integer count = partialWindowTuples.get(value);
                    partialWindowTuples.put(value, count + 1);
                } else {
                    partialWindowTuples.put(value, 1);
                }
            }
            if (crossedBoundary) {
                break;
            }
        }
    }
}

Example 80 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project apex-malhar by apache.

the class KafkaSinglePortExactlyOnceOutputOperator method getPartitionsAndOffsets.

private Map<Integer, Long> getPartitionsAndOffsets(boolean latest) throws ExecutionException, InterruptedException {
    List<PartitionInfo> partitionInfoList = consumer.partitionsFor(getTopic());
    List<TopicPartition> topicPartitionList = new java.util.ArrayList<>();
    for (PartitionInfo partitionInfo : partitionInfoList) {
        topicPartitionList.add(new TopicPartition(getTopic(), partitionInfo.partition()));
    }
    Map<Integer, Long> parttionsAndOffset = new HashMap<>();
    consumer.assign(topicPartitionList);
    for (PartitionInfo partitionInfo : partitionInfoList) {
        try {
            TopicPartition topicPartition = new TopicPartition(getTopic(), partitionInfo.partition());
            if (latest) {
                consumer.seekToEnd(topicPartition);
            } else {
                consumer.seekToBeginning(topicPartition);
            }
            parttionsAndOffset.put(partitionInfo.partition(), consumer.position(topicPartition));
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
    }
    return parttionsAndOffset;
}