Examples with UnsupportedControlPortActionException com.ibm.streamsx.kafka.UnsupportedControlPortActionException used on opensource projects

Example 1 with UnsupportedControlPortActionException

use of com.ibm.streamsx.kafka.UnsupportedControlPortActionException in project streamsx.kafka by IBMStreams.

the class CrKafkaStaticAssignConsumerClient method processControlPortActionEvent.

/**
 * @see com.ibm.streamsx.kafka.clients.consumer.AbstractKafkaConsumerClient#processControlPortActionEvent(com.ibm.streamsx.kafka.clients.consumer.ControlPortAction)
 */
@Override
protected void processControlPortActionEvent(ControlPortAction update) {
    // trace with info. to see this method call is important, and it happens not frequently.
    logger.info("processControlPortActionEvent(): update = " + update);
    // create a map of current topic partitions and their offsets
    Map<TopicPartition, Long> /* offset */
    currentTopicPartitionOffsets = new HashMap<TopicPartition, Long>();
    Set<TopicPartition> topicPartitions = getConsumer().assignment();
    topicPartitions.forEach(tp -> currentTopicPartitionOffsets.put(tp, getConsumer().position(tp)));
    final ControlPortActionType actionType = update.getActionType();
    switch(actionType) {
        case ADD_ASSIGNMENT:
            try {
                update.getTopicPartitionOffsetMap().forEach((tp, offset) -> {
                    // offset can be -2, -1, or a valid offset o >= 0
                    // -2 means 'seek to beginning', -1 means 'seek to end'
                    currentTopicPartitionOffsets.put(tp, offset);
                });
                assignToPartitionsWithOffsets(currentTopicPartitionOffsets);
                synchronized (offsetManager) {
                    // avoid concurrent access with tuple submission thread
                    // update offset manager: add topics or updates their partition lists
                    offsetManager.updateTopics(currentTopicPartitionOffsets.keySet());
                    // save the consumer offsets after moving it's position
                    offsetManager.savePositionFromCluster();
                    createJcpCvFromOffsetManagerl();
                }
                break;
            } catch (Exception e) {
                throw new KafkaOperatorRuntimeException(e.getMessage(), e);
            }
        case REMOVE_ASSIGNMENT:
            try {
                update.getTopicPartitionOffsetMap().forEach((tp, offset) -> {
                    currentTopicPartitionOffsets.remove(tp);
                });
                // remove messages of removed topic partitions from the message queue; we would not be able to commit them later
                getMessageQueue().removeIf(record -> belongsToPartition(record, update.getTopicPartitionOffsetMap().keySet()));
                // remove removed partitions from offset manager.
                synchronized (offsetManager) {
                    // avoid concurrent access with tuple submission thread
                    update.getTopicPartitionOffsetMap().forEach((tp, offset) -> {
                        offsetManager.remove(tp.topic(), tp.partition());
                    });
                    assignToPartitionsWithOffsets(currentTopicPartitionOffsets);
                    // save the consumer offsets after moving it's position
                    offsetManager.savePositionFromCluster();
                    createJcpCvFromOffsetManagerl();
                }
                break;
            } catch (Exception e) {
                throw new KafkaOperatorRuntimeException(e.getMessage(), e);
            }
        default:
            throw new UnsupportedControlPortActionException("processControlPortActionEvent(): action: " + actionType + " not supported by this client: " + getThisClassName());
    }
}

Example 2 with UnsupportedControlPortActionException

use of com.ibm.streamsx.kafka.UnsupportedControlPortActionException in project streamsx.kafka by IBMStreams.

the class NonCrKafkaConsumerGroupClient method processControlPortActionEvent.

/**
 * Changes the subscription of the consumer via control port.
 *
 * @see com.ibm.streamsx.kafka.clients.consumer.AbstractKafkaConsumerClient#processControlPortActionEvent(com.ibm.streamsx.kafka.clients.consumer.ControlPortAction)
 */
@Override
protected void processControlPortActionEvent(ControlPortAction action) {
    try {
        final ControlPortActionType actionType = action.getActionType();
        if (actionType == ControlPortActionType.ADD_SUBSCRIPTION || actionType == ControlPortActionType.REMOVE_SUBSCRIPTION) {
            trace.info("action: " + action);
        } else if (trace.isDebugEnabled()) {
            trace.debug("action: " + action);
        }
        final Set<String> oldSubscription = getConsumer().subscription();
        final Set<String> newSubscription = new HashSet<>(oldSubscription);
        trace.info("current topic subscription: " + newSubscription);
        boolean subscriptionChanged = false;
        switch(actionType) {
            case ADD_SUBSCRIPTION:
                action.getTopics().forEach(tpc -> {
                    newSubscription.add(tpc);
                });
                break;
            case REMOVE_SUBSCRIPTION:
                action.getTopics().forEach(tpc -> {
                    newSubscription.remove(tpc);
                });
                break;
            default:
                throw new UnsupportedControlPortActionException("processControlPortActionEvent(): action: " + actionType + " not supported by this client: " + getThisClassName());
        }
        subscriptionChanged = !newSubscription.equals(oldSubscription);
        if (!subscriptionChanged) {
            trace.info("Subscriptiopn is unchanged: " + newSubscription);
        } else {
            if (newSubscription.size() > 0) {
                trace.info("Subscription changed. New subscription: " + newSubscription);
            } else {
                // With Kafka client 2.3, no partition rebalance happened when we only unsubscribe, so that the
                // onPartitionsRevoked callback has not been called, where we usually commit offsets before we give
                // away partitions.
                // With Kafka client 2.5.1, the things are different. The onPartitionsRevoked callback is called,
                // so that we could commit the offsets there and remove the following code block, but it is also safe,
                // to preserve the old logic and commit now.
                trace.info("Unsubscribing all topics. Going to commit offsets.");
                // remove the content of the queue. It contains uncommitted messages.
                getMessageQueue().clear();
                OffsetManager offsetManager = getOffsetManager();
                try {
                    awaitMessageQueueProcessed();
                    // the post-condition is, that all messages from the queue have submitted as
                    // tuples and its offsets +1 are stored in OffsetManager.
                    final boolean commitSync = true;
                    final boolean commitPartitionWise = false;
                    CommitInfo offsets = new CommitInfo(commitSync, commitPartitionWise);
                    synchronized (offsetManager) {
                        Set<TopicPartition> partitionsInOffsetManager = offsetManager.getMappedTopicPartitions();
                        Set<TopicPartition> currentAssignment = getAssignedPartitions();
                        for (TopicPartition tp : partitionsInOffsetManager) {
                            if (currentAssignment.contains(tp)) {
                                offsets.put(tp, offsetManager.getOffset(tp.topic(), tp.partition()));
                            }
                        }
                    }
                    if (!offsets.isEmpty()) {
                        commitOffsets(offsets);
                    }
                    // reset the counter for periodic commit
                    resetCommitPeriod(System.currentTimeMillis());
                } catch (InterruptedException | RuntimeException e) {
                // Ignore InterruptedException, RuntimeException from commitOffsets is already traced.
                }
                // avoid committing offsets in onPartitionsRevoked (if called)
                offsetManager.clear();
            }
            subscribe(newSubscription, this);
            // getChkptContext().getKind() is not reported properly. Streams Build 20180710104900 (4.3.0.0) never returns OPERATOR_DRIVEN
            if (getCheckpointKind() == Kind.OPERATOR_DRIVEN) {
                trace.info("initiating checkpointing with current topic subscription");
                // createCheckpoint() throws IOException
                boolean result = getChkptContext().createCheckpoint();
                trace.info("createCheckpoint() result: " + result);
            }
        }
    } catch (Exception e) {
        trace.error(e.getLocalizedMessage(), e);
        throw new KafkaOperatorRuntimeException(e.getMessage(), e);
    }
}

Example 3 with UnsupportedControlPortActionException

use of com.ibm.streamsx.kafka.UnsupportedControlPortActionException in project streamsx.kafka by IBMStreams.

the class NonCrKafkaConsumerClient method processControlPortActionEvent.

/**
 * @see com.ibm.streamsx.kafka.clients.consumer.AbstractKafkaConsumerClient#processControlPortActionEvent(com.ibm.streamsx.kafka.clients.consumer.ControlPortAction)
 */
@Override
protected void processControlPortActionEvent(ControlPortAction action) {
    try {
        final ControlPortActionType actionType = action.getActionType();
        if (actionType == ControlPortActionType.ADD_ASSIGNMENT || actionType == ControlPortActionType.REMOVE_ASSIGNMENT) {
            trace.info("action: " + action);
        } else if (trace.isDebugEnabled()) {
            trace.debug("action: " + action);
        }
        // create a map of current topic partitions and their fetch offsets for next record
        Map<TopicPartition, Long> /* offset */
        currentTopicPartitionOffsets = new HashMap<TopicPartition, Long>();
        Set<TopicPartition> topicPartitions = getConsumer().assignment();
        topicPartitions.forEach(tp -> currentTopicPartitionOffsets.put(tp, getConsumer().position(tp)));
        boolean doNewAssign = false;
        switch(actionType) {
            case ADD_ASSIGNMENT:
                action.getTopicPartitionOffsetMap().forEach((tp, offset) -> {
                    // offset can be -2, -1, or a valid offset o >= 0
                    // -2 means 'seek to beginning', -1 means 'seek to end'
                    currentTopicPartitionOffsets.put(tp, offset);
                });
                doNewAssign = currentTopicPartitionOffsets.size() > 0;
                if (!doNewAssign) {
                    trace.info("topic partition assignment unchanged: " + currentTopicPartitionOffsets);
                } else {
                    assignToPartitionsWithOffsets(currentTopicPartitionOffsets);
                    trace.info("assigned partitions after ADD: " + currentTopicPartitionOffsets);
                    // No need to update offset manager here, like adding topics, etc.
                    // Missing topics in the offset manager are auto-created
                    CommitInfo commits = new CommitInfo(true, false);
                    // Immediately commit the fetch offsets of _only_the_added_ topic partitions
                    action.getTopicPartitionOffsetMap().forEach((tp, offset) -> {
                        // do not put 'offset' into the commits; 'offset' can be -1 or -2 for 'end' or 'begin'
                        commits.put(tp, getConsumer().position(tp));
                    });
                    commitOffsets(commits);
                    trace.info("committed offsets of the added topic partitions: " + commits);
                }
                break;
            case REMOVE_ASSIGNMENT:
                // x 1. remove messages of the removed topic partitions from the queue - they are all uncommitted
                // x 2. wait that the queue gets processed - awaitMessageQueueProcessed();
                // x 3. commit the offsets of the removed topic partitions
                // x 4. remove the unassigned topic partitions from the offsetManager (or simply clear?)
                // x 5. update the partition assignment in the consumer
                // remove messages of removed topic partitions from the message queue
                getMessageQueue().removeIf(record -> belongsToPartition(record, action.getTopicPartitionOffsetMap().keySet()));
                awaitMessageQueueProcessed();
                // now the offset manager can be cleaned without the chance that the removed partition(s) re-appear after tuple submission
                // remove removed partitions from offset manager. We can't commit offsets for those partitions we are not assigned any more.
                // the post-condition is, that all messages from the queue have submitted as
                // tuples and its offsets +1 are stored in OffsetManager.
                final boolean commitSync = true;
                final boolean commitPartitionWise = false;
                CommitInfo commitOffsets = new CommitInfo(commitSync, commitPartitionWise);
                OffsetManager offsetManager = getOffsetManager();
                synchronized (offsetManager) {
                    for (TopicPartition tp : action.getTopicPartitionOffsetMap().keySet()) {
                        // make sure that we commit only partitions that are assigned
                        if (currentTopicPartitionOffsets.containsKey(tp)) {
                            doNewAssign = true;
                            long offset = offsetManager.getOffset(tp.topic(), tp.partition());
                            // offset is -1 if there is no mapping from topic partition to offset
                            if (offset >= 0)
                                commitOffsets.put(tp, offset);
                            currentTopicPartitionOffsets.remove(tp);
                        }
                        offsetManager.remove(tp.topic(), tp.partition());
                    }
                }
                if (!commitOffsets.isEmpty()) {
                    commitOffsets(commitOffsets);
                }
                // we can end up here with an empty map after removal of assignments.
                if (doNewAssign) {
                    assignToPartitionsWithOffsets(currentTopicPartitionOffsets);
                    trace.info("assigned partitions after REMOVE: " + currentTopicPartitionOffsets);
                } else {
                    trace.info("topic partition assignment unchanged: " + currentTopicPartitionOffsets);
                }
                break;
            default:
                throw new UnsupportedControlPortActionException("processControlPortActionEvent(): action: " + actionType + " not supported by this client: " + getThisClassName());
        }
        // getChkptContext().getKind() is not reported properly. Streams Build 20180710104900 (4.3.0.0) never returns OPERATOR_DRIVEN
        if (doNewAssign && getCheckpointKind() == Kind.OPERATOR_DRIVEN) {
            trace.info("initiating checkpointing with current partition assignment");
            // createCheckpoint() throws IOException
            boolean result = getChkptContext().createCheckpoint();
            trace.info("createCheckpoint() result: " + result);
        }
    } catch (Exception e) {
        trace.error(e.getLocalizedMessage(), e);
        throw new KafkaOperatorRuntimeException(e.getMessage(), e);
    }
}