Examples with OffsetManager com.ibm.streamsx.kafka.clients.OffsetManager used on opensource projects

Example 1 with OffsetManager

use of com.ibm.streamsx.kafka.clients.OffsetManager in project streamsx.kafka by IBMStreams.

the class KafkaConsumerClient method saveOffsetManagerToJCP.

private void saveOffsetManagerToJCP() throws Exception {
    ControlPlaneContext controlPlaneContext = operatorContext.getOptionalContext(ControlPlaneContext.class);
    offsetManagerCV = controlPlaneContext.createStringControlVariable(OffsetManager.class.getName(), false, serializeObject(offsetManager));
    OffsetManager mgr = getDeserializedOffsetManagerCV();
    logger.debug("Retrieved value for offsetManagerCV=" + mgr);
}

Example 2 with OffsetManager

use of com.ibm.streamsx.kafka.clients.OffsetManager in project streamsx.kafka by IBMStreams.

the class NonCrKafkaConsumerGroupClient method onPartitionsAssigned.

/**
 * Callback method of the ConsumerRebalanceListener.
 * For incremental cooperative rebalancing we must assume that we get only the incrementally added partitions,
 * not the complete assignment. The callback can also be called without prior {@link #onPartitionsRevoked(Collection)}.
 * The eager rebalancing protocol passes always the complete partition assignment after {@link #onPartitionsRevoked(Collection)}.
 *
 * @see org.apache.kafka.clients.consumer.ConsumerRebalanceListener#onPartitionsAssigned(java.util.Collection)
 * @param partitions The added topic partitions
 */
@Override
public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
    trace.info("onPartitionsAssigned: new/added partition assignment = " + partitions);
    Set<TopicPartition> newAssignment = addAssignedPartitions(partitions);
    if (partitions.size() > 0) {
        trace.info("onPartitionsAssigned: changed partition assignment: " + newAssignment);
    } else {
        trace.info("onPartitionsAssigned: partition assignment is unchanged: " + newAssignment);
    }
    getOperatorContext().getMetrics().getCustomMetric(N_PARTITION_REBALANCES).increment();
    try {
        // override the fetch offset according to initialStartPosition for
        // those partitions, which are never committed within the group
        final StartPosition startPos = getInitialStartPosition();
        for (TopicPartition tp : partitions) {
            switch(startPos) {
                case Default:
                    break;
                case Beginning:
                case End:
                    if (!isCommittedForPartition(tp)) {
                        seekToPosition(tp, startPos);
                    }
                    break;
                case Time:
                    if (!isCommittedForPartition(tp)) {
                        seekToTimestamp(tp, this.initialStartTimestamp);
                    }
                    break;
                default:
                    // unsupported start position, like 'Offset',  is already treated by initialization checks
                    final String msg = MsgFormatter.format("onPartitionsAssigned(): {0} does not support startPosition {1}.", getThisClassName(), getInitialStartPosition());
                    trace.error(msg);
                    throw new RuntimeException(msg);
            }
        }
        try {
            checkSpaceInMessageQueueAndPauseFetching(true);
        } catch (IllegalStateException e) {
        // IllegalStateException cannot happen
        // On Interruption, do nothing
        }
    } catch (InterruptedException e) {
        trace.debug("onPartitionsAssigned(): thread interrupted");
    }
    if (partitions.size() > 0) {
        OffsetManager offsetManager = getOffsetManager();
        synchronized (offsetManager) {
            // cleanup the offset for added partitions we have not owned before.
            for (TopicPartition tp : partitions) {
                offsetManager.remove(tp.topic(), tp.partition());
            }
        }
    }
}

Example 3 with OffsetManager

use of com.ibm.streamsx.kafka.clients.OffsetManager in project streamsx.kafka by IBMStreams.

the class CrKafkaConsumerGroupClient method createSeekOffsetMap.

/**
 * The seek offsets are created with following algorithm from the checkpoint:
 * <ul>
 * <li>read the contributing operator names from the checkpoint (operator names of the consumer group)
 * <li>read the seek offsets from the checkpoint.
 *     These are the offsets of only those partitions the consumer was assigned at checkpoint time.</li>
 * <li>send the offsets of the prior partitions together with the number of operators and the own operator name to the CrGroupCoordinator MXBean.
 *     The other consumer operators will also send their prior partition-to-offset mappings, and and their dsitinct operator name.</li>
 * <li>wait for the JMX notification that the partition-to-offset map has merged</li>
 * <li>fetch the merged map from the MX bean so that the operator has the seek offsets of all partitions of
 *     all topics (the group's view) and store this in the 'seekOffsetMap' member variable.</li>
 * </ul>
 * @param checkpoint
 * @throws InterruptedException
 */
@SuppressWarnings("unchecked")
private void createSeekOffsetMap(Checkpoint checkpoint) throws InterruptedException {
    final String operatorName = getOperatorContext().getName();
    long chkptSeqId = checkpoint.getSequenceId();
    int resetAttempt = getCrContext().getResetAttempt();
    MergeKey key = new MergeKey(chkptSeqId, resetAttempt);
    trace.info(MsgFormatter.format("createSeekOffsetMap() [{0}] - entering. chkptSeqId = {1,number,#}, resetAttempt = {2}", state, chkptSeqId, resetAttempt));
    try {
        final ObjectInputStream inputStream = checkpoint.getInputStream();
        final String myOperatorNameInCkpt = (String) inputStream.readObject();
        Set<String> contributingOperators = (Set<String>) inputStream.readObject();
        OffsetManager offsMgr = (OffsetManager) inputStream.readObject();
        trace.info(MsgFormatter.format("createSeekOffsetMap() - merging {0} operator checkpoints into a single group checkpoint", contributingOperators.size()));
        if (trace.isEnabledFor(DEBUG_LEVEL)) {
            trace.log(DEBUG_LEVEL, MsgFormatter.format("createSeekOffsetMap(): myOperatorName read from checkpoint: {0}", myOperatorNameInCkpt));
            trace.log(DEBUG_LEVEL, MsgFormatter.format("createSeekOffsetMap(): contributingOperators read from checkpoint: {0}", contributingOperators));
            trace.log(DEBUG_LEVEL, MsgFormatter.format("createSeekOffsetMap(): offset manager read from checkpoint: {0}", offsMgr));
        }
        if (!operatorName.equals(myOperatorNameInCkpt)) {
            trace.warn(MsgFormatter.format("Operator name in checkpoint ({0}) differs from current operator name: {1}", myOperatorNameInCkpt, operatorName));
        }
        if (!contributingOperators.contains(operatorName)) {
            final String msg = MsgFormatter.format("This operator''s name ({0}) not found in contributing operator names: {1}", operatorName, contributingOperators);
            trace.error(msg);
            throw new KafkaOperatorResetFailedException(msg);
        }
        trace.info(MsgFormatter.format("contributing {0} partition => offset mappings to the group''s checkpoint.", offsMgr.size()));
        if (contributingOperators.size() == 1) {
            trace.info("this single operator participated in consumer group at checkpoint time. Checkpoint merge and distribution via MXBean disabled.");
            assert (contributingOperators.contains(operatorName));
            initSeekOffsetMap();
            for (TopicPartition tp : offsMgr.getMappedTopicPartitions()) {
                final String topic = tp.topic();
                final int partition = tp.partition();
                final Long offset = offsMgr.getOffset(topic, partition);
                this.seekOffsetMap.put(tp, offset);
            }
        } else {
            // send checkpoint data to CrGroupCoordinator MXBean and wait for the notification
            // to fetch the group's complete checkpoint. Then, process the group's checkpoint.
            Map<CrConsumerGroupCoordinator.TP, Long> partialOffsetMap = new HashMap<>();
            for (TopicPartition tp : offsMgr.getMappedTopicPartitions()) {
                final String topic = tp.topic();
                final int partition = tp.partition();
                final Long offset = offsMgr.getOffset(topic, partition);
                partialOffsetMap.put(new TP(topic, partition), offset);
            }
            trace.info(MsgFormatter.format("Merging my group''s checkpoint contribution: partialOffsetMap = {0}, myOperatorName = {1}", partialOffsetMap, operatorName));
            this.crGroupCoordinatorMxBean.mergeConsumerCheckpoint(chkptSeqId, resetAttempt, contributingOperators.size(), partialOffsetMap, operatorName);
            // check JMX notification and wait for notification
            jmxNotificationConditionLock.lock();
            long waitStartTime = System.currentTimeMillis();
            // increase timeout exponentially with every reset attempt by 20%
            // long timeoutMillis = (long)(Math.pow (1.2, resetAttempt) * (double)timeouts.getJmxResetNotificationTimeout());
            long timeoutMillis = timeouts.getJmxResetNotificationTimeout();
            boolean waitTimeLeft = true;
            int nWaits = 0;
            long timeElapsed = 0;
            trace.log(DEBUG_LEVEL, MsgFormatter.format("checking receiption of JMX notification {0} for sequenceId {1}. timeout = {2,number,#} ms.", CrConsumerGroupCoordinatorMXBean.MERGE_COMPLETE_NTF_TYPE, key, timeoutMillis));
            while (!jmxMergeCompletedNotifMap.containsKey(key) && waitTimeLeft) {
                long remainingTime = timeoutMillis - timeElapsed;
                waitTimeLeft = remainingTime > 0;
                if (waitTimeLeft) {
                    if (nWaits++ % 50 == 0)
                        trace.log(DEBUG_LEVEL, MsgFormatter.format("waiting for JMX notification {0} for sequenceId {1}. Remaining time = {2,number,#} of {3,number,#} ms", CrConsumerGroupCoordinatorMXBean.MERGE_COMPLETE_NTF_TYPE, key, remainingTime, timeoutMillis));
                    jmxNotificationCondition.await(100, TimeUnit.MILLISECONDS);
                }
                timeElapsed = System.currentTimeMillis() - waitStartTime;
            }
            CrConsumerGroupCoordinator.CheckpointMerge merge = jmxMergeCompletedNotifMap.get(key);
            jmxNotificationConditionLock.unlock();
            if (merge == null) {
                final String msg = MsgFormatter.format("timeout receiving {0} JMX notification for {1} from MXBean {2} in JCP. Current timeout is {3,number,#} milliseconds.", CrConsumerGroupCoordinatorMXBean.MERGE_COMPLETE_NTF_TYPE, key, crGroupCoordinatorMXBeanName, timeoutMillis);
                trace.error(msg);
                throw new KafkaOperatorResetFailedException(msg);
            } else {
                trace.info(MsgFormatter.format("waiting for JMX notification for sequenceId {0} took {1} ms", key, timeElapsed));
            }
            Map<TP, Long> mergedOffsetMap = merge.getConsolidatedOffsetMap();
            trace.info("reset offsets (group's checkpoint) received from MXBean: " + mergedOffsetMap);
            initSeekOffsetMap();
            mergedOffsetMap.forEach((tp, offset) -> {
                this.seekOffsetMap.put(new TopicPartition(tp.getTopic(), tp.getPartition()), offset);
            });
        }
    } catch (InterruptedException e) {
        trace.log(DEBUG_LEVEL, "createSeekOffsetMap(): interrupted waiting for the JMX notification");
        return;
    } catch (IOException | ClassNotFoundException e) {
        trace.error("reset failed: " + e.getLocalizedMessage());
        throw new KafkaOperatorResetFailedException(MsgFormatter.format("resetting operator {0} to checkpoint sequence ID {1} failed: {2}", getOperatorContext().getName(), chkptSeqId, e.getLocalizedMessage()), e);
    }
    trace.log(DEBUG_LEVEL, "createSeekOffsetMap(): seekOffsetMap = " + this.seekOffsetMap);
}

Example 4 with OffsetManager

use of com.ibm.streamsx.kafka.clients.OffsetManager in project streamsx.kafka by IBMStreams.

the class CrKafkaStaticAssignConsumerClient method processResetEvent.

/**
 * Resets the client to a previous state.
 * @param checkpoint the checkpoint that contains the previous state.
 */
@Override
protected void processResetEvent(Checkpoint checkpoint) {
    logger.log(DEBUG_LEVEL, "processResetEvent() - entering. seq = " + checkpoint.getSequenceId());
    try {
        clearDrainBuffer();
        getMessageQueue().clear();
        final OffsetManager ofsm = (OffsetManager) checkpoint.getInputStream().readObject();
        // $NON-NLS-1$
        logger.log(DEBUG_LEVEL, "offsetManager from checkpoint = " + ofsm);
        offsetManager.putOffsets(ofsm);
        // $NON-NLS-1$
        logger.log(DEBUG_LEVEL, "offsetManager after applying checkpoint = " + offsetManager);
        refreshFromCluster();
        this.nSubmittedRecords = 0;
    } catch (Exception e) {
        throw new RuntimeException(e.getLocalizedMessage(), e);
    }
    logger.log(DEBUG_LEVEL, "processResetEvent() - exiting");
}

Example 5 with OffsetManager

use of com.ibm.streamsx.kafka.clients.OffsetManager in project streamsx.kafka by IBMStreams.

the class CrKafkaStaticAssignConsumerClient method createJcpCvFromOffsetManagerl.

/**
 * Creates an operator-scoped JCP control variable and stores the Offset manager in serialized format.
 * @throws Exception
 */
private void createJcpCvFromOffsetManagerl() throws Exception {
    logger.log(DEBUG_LEVEL, "createJcpCvFromOffsetManagerl(). offsetManager = " + offsetManager);
    offsetManagerCV = getJcpContext().createStringControlVariable(OffsetManager.class.getName(), false, serializeObject(offsetManager));
    OffsetManager mgr = getDeserializedOffsetManagerCV();
    logger.log(DEBUG_LEVEL, "Retrieved value for offsetManagerCV = " + mgr);
}