Examples with ConsumerClient com.ibm.streamsx.kafka.clients.consumer.ConsumerClient used on opensource projects

Example 6 with ConsumerClient

use of com.ibm.streamsx.kafka.clients.consumer.ConsumerClient in project streamsx.kafka by IBMStreams.

the class AbstractKafkaConsumerOperator method checkpoint.

@Override
public void checkpoint(Checkpoint checkpoint) throws Exception {
    // $NON-NLS-1$ //$NON-NLS-2$
    logger.log(DEBUG_LEVEL, ">>> CHECKPOINT (ckpt id=" + checkpoint.getSequenceId() + ")");
    final ConsumerClient consumer = consumerRef.get();
    checkpoint.getOutputStream().writeInt(consumer.getImplementationMagic());
    if (consumer.isProcessing()) {
        consumer.onCheckpoint(checkpoint);
    }
}

Example 7 with ConsumerClient

use of com.ibm.streamsx.kafka.clients.consumer.ConsumerClient in project streamsx.kafka by IBMStreams.

the class AbstractKafkaConsumerOperator method produceTuples.

private void produceTuples() throws Exception {
    if (crContext != null && resettingLatch != null) {
        // $NON-NLS-1$
        logger.log(DEBUG_LEVEL, "Defer tuple submission until reset finishes. Waiting ...");
        try {
            resettingLatch.await();
        } catch (InterruptedException e) {
            // shutdown occurred in the middle of CR reset, finish gracefully
            return;
        }
    }
    final ConsumerClient consumerInitial = consumerRef.get();
    if (consumerInitial.isSubscribedOrAssigned()) {
        consumerInitial.sendStartPollingEvent();
    }
    /*
         * Shutdown implementation:
         * On shutdown, all threads get interrupted and throw InterruptedException, which must be caught and handled.
         * 
         * When we catch InterruptedException, the method is immediately left by return statement. When we only leave
         * the while-loop, we run into `consumer.sendStopPollingEvent();`, which contains a wait, that another thread processes
         * the event. This will most likely not happen because this thread also has been interrupted and finished working.
         */
    while (!shutdown.get()) {
        if (crContext != null) {
            try {
                // logger.trace("Acquiring consistent region permit..."); //$NON-NLS-1$
                crContext.acquirePermit();
            } catch (InterruptedException e) {
                // shutdown occured waiting for permit, finish gracefully
                // $NON-NLS-1$
                logger.debug(Messages.getString("ERROR_ACQUIRING_PERMIT", e.getLocalizedMessage()));
                return;
            }
        }
        try {
            ConsumerClient consumer = consumerRef.get();
            // Any exceptions except InterruptedException thrown here are propagated to the caller
            // Make timeout for 'getNextRecord' not too high as it influences the granularity of time based offset commit
            ConsumerRecord<?, ?> record = consumer.getNextRecord(100, TimeUnit.MILLISECONDS);
            if (record != null) {
                submitRecord(record);
                consumer.postSubmit(record);
            }
        } catch (InterruptedException ie) {
            logger.debug("Queue processing thread interrupted", ie);
            return;
        } finally {
            if (crContext != null) {
                // $NON-NLS-1$
                if (logger.isDebugEnabled())
                    logger.debug("Releasing consistent region permit...");
                crContext.releasePermit();
            }
        }
    }
    try {
        consumerRef.get().sendStopPollingEvent();
    } catch (InterruptedException ie) {
        // interrupted during shutdown
        return;
    }
}

Example 8 with ConsumerClient

use of com.ibm.streamsx.kafka.clients.consumer.ConsumerClient in project streamsx.kafka by IBMStreams.

the class AbstractKafkaConsumerOperator method initialize.

@Override
public void initialize(OperatorContext context) throws Exception {
    synchronized (monitor) {
        // Must call super.initialize(context) to correctly setup an operator.
        super.initialize(context);
        logger.info(// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
        "Operator " + context.getName() + " initializing in PE: " + context.getPE().getPEId() + " in Job: " + context.getPE().getJobId());
        shutdown = new AtomicBoolean(false);
        StreamSchema outputSchema = context.getStreamingOutputs().get(0).getStreamSchema();
        outputMessageAttrIdx = outputSchema.getAttributeIndex(outputMessageAttrName);
        outputKeyAttrIdx = outputSchema.getAttributeIndex(outputKeyAttrName);
        outputTopicAttrIdx = outputSchema.getAttributeIndex(outputTopicAttrName);
        outputTimetampAttrIdx = outputSchema.getAttributeIndex(outputMessageTimestampAttrName);
        outputPartitionAttrIdx = outputSchema.getAttributeIndex(outputPartitionAttrName);
        outputOffsetAttrIdx = outputSchema.getAttributeIndex(outputOffsetAttrName);
        Class<?> keyClass = outputKeyAttrIdx >= 0 ? getAttributeType(context.getStreamingOutputs().get(0), outputKeyAttrName) : // default to String.class for key type
        String.class;
        Class<?> valueClass = getAttributeType(context.getStreamingOutputs().get(0), outputMessageAttrName);
        KafkaOperatorProperties kafkaProperties = getKafkaProperties();
        if (this.startPosition == StartPosition.Time && !context.getParameterNames().contains(START_TIME_PARAM)) {
            throw new KafkaConfigurationException(Messages.getString("START_TIME_PARAM_NOT_FOUND"));
        }
        if (this.startPosition == StartPosition.Offset && !context.getParameterNames().contains(START_OFFSET_PARAM)) {
            throw new KafkaConfigurationException(Messages.getString("START_OFFSET_PARAM_NOT_FOUND"));
        }
        // set the group ID property if the groupId parameter is specified
        if (groupId != null && !groupId.isEmpty()) {
            kafkaProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId);
        }
        final boolean hasInputPorts = context.getStreamingInputs().size() > 0;
        final String gid = kafkaProperties.getProperty(ConsumerConfig.GROUP_ID_CONFIG);
        this.groupIdSpecified = gid != null && !gid.trim().isEmpty();
        logger.log(DEBUG_LEVEL, "group-ID specified: " + this.groupIdSpecified);
        if (crContext != null) {
            commitMode = CommitMode.ConsistentRegionDrain;
        } else {
            final Set<String> parameterNames = context.getParameterNames();
            commitMode = parameterNames.contains(COMMIT_COUNT_PARAM) ? CommitMode.TupleCount : CommitMode.Time;
        }
        if (this.staticGroupMember) {
            // calculate a unique group.instance.id that is consistent across operator restarts
            final ProcessingElement pe = context.getPE();
            final int iidH = pe.getInstanceId().hashCode();
            final int opnH = context.getName().hashCode();
            final String groupInstanceId = MsgFormatter.format("i{0}-o{1}", (iidH < 0 ? "N" + (-iidH) : "P" + iidH), (opnH < 0 ? "N" + (-opnH) : "P" + opnH));
            logger.info("Generated group.instance.id: " + groupInstanceId);
            kafkaProperties.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, groupInstanceId);
        }
        // create the builders for the consumer clients
        if (crContext == null) {
            this.groupEnabledClientBuilder = new NonCrKafkaConsumerGroupClient.Builder().setOperatorContext(context).setKafkaProperties(kafkaProperties).setKeyClass(keyClass).setValueClass(valueClass).setSingleTopic(this.topics != null && this.topics.size() == 1).setPollTimeout(this.consumerPollTimeout).setInitialStartPosition(this.startPosition).setCommitMode(commitMode).setCommitPeriod(commitPeriod).setCommitCount(commitCount);
            this.staticAssignClientBuilder = new NonCrKafkaConsumerClient.Builder().setOperatorContext(context).setKafkaProperties(kafkaProperties).setKeyClass(keyClass).setValueClass(valueClass).setPollTimeout(this.consumerPollTimeout).setInitialStartPosition(this.startPosition).setCommitMode(commitMode).setCommitPeriod(commitPeriod).setCommitCount(commitCount);
        } else {
            // CR
            this.groupEnabledClientBuilder = new CrKafkaConsumerGroupClient.Builder().setOperatorContext(context).setKafkaProperties(kafkaProperties).setKeyClass(keyClass).setValueClass(valueClass).setPollTimeout(this.consumerPollTimeout).setSingleTopic(this.topics != null && this.topics.size() == 1).setTriggerCount(this.triggerCount).setInitialStartPosition(this.startPosition).setInitialStartTimestamp(this.startTime);
            this.staticAssignClientBuilder = new CrKafkaStaticAssignConsumerClient.Builder().setOperatorContext(context).setKafkaProperties(kafkaProperties).setKeyClass(keyClass).setValueClass(valueClass).setPollTimeout(this.consumerPollTimeout).setTriggerCount(this.triggerCount);
        }
        magics.put(this.staticAssignClientBuilder.getImplementationMagic(), this.staticAssignClientBuilder);
        magics.put(this.groupEnabledClientBuilder.getImplementationMagic(), this.groupEnabledClientBuilder);
        final ConsumerClientBuilder builder;
        if (hasInputPorts) {
            if (crContext != null) {
                // in CR, we do not groupManagement with input port:
                builder = this.staticAssignClientBuilder;
            } else {
                // not in CR: select the right builder in checkpoint reset or on first partition/topic addition
                builder = new DummyConsumerClient.Builder().setOperatorContext(context).setKafkaProperties(kafkaProperties);
                magics.put(builder.getImplementationMagic(), builder);
            }
        } else {
            boolean groupManagementEnabled;
            if (Features.ENABLE_GROUP_MANAGEMENT_NO_USER_GROUP_ID) {
                groupManagementEnabled = this.partitions == null || this.partitions.isEmpty();
            } else {
                // legacy (2.x) behavior
                groupManagementEnabled = this.groupIdSpecified && (this.partitions == null || this.partitions.isEmpty());
            }
            if (this.groupIdSpecified && !groupManagementEnabled) {
                if (this.partitions != null && !this.partitions.isEmpty()) {
                    logger.warn(MsgFormatter.format("The group.id ''{0}'' is specified. The ''{1}'' operator " + "will NOT participate in a consumer group because partitions to consume are specified.", gid, context.getName()));
                }
            }
            // when group management is disabled and no input port is configured, we must not subscribe with pattern
            // When we are here it is already guaranteed that we have one of the 'topic' or 'pattern' parameter
            final boolean p = this.pattern != null;
            final boolean t = this.topics != null;
            assert ((p && !t) || (t && !p));
            if (!groupManagementEnabled && p) {
                final String msg = Messages.getString("PATTERN_SUBSCRIPTION_REQUIRES_GROUP_MGT", PATTERN_PARAM, context.getName(), context.getKind());
                logger.error(msg);
                throw new KafkaConfigurationException(msg);
            }
            builder = groupManagementEnabled ? this.groupEnabledClientBuilder : this.staticAssignClientBuilder;
        }
        ConsumerClient client = builder.build();
        consumerRef = new AtomicReference<>(client);
        logger.info(MsgFormatter.format("consumer client {0} created", client.getClass().getName()));
        try {
            client.startConsumer();
        } catch (KafkaClientInitializationException e) {
            e.printStackTrace();
            logger.error(e.getLocalizedMessage(), e);
            logger.error("root cause: " + e.getRootCause());
            throw e;
        }
        // input port not used, so topic or pattern must be defined
        if (!hasInputPorts) {
            if (this.topics != null) {
                final boolean registerAsInput = true;
                registerForDataGovernance(context, topics, registerAsInput);
                switch(startPosition) {
                    case Time:
                        client.subscribeToTopicsWithTimestamp(topics, partitions, startTime);
                        break;
                    case Offset:
                        client.subscribeToTopicsWithOffsets(topics.get(0), partitions, startOffsets);
                        break;
                    default:
                        client.subscribeToTopics(topics, partitions, startPosition);
                }
            } else {
                switch(startPosition) {
                    case Time:
                        client.subscribeToTopicsWithTimestamp(pattern, startTime);
                        break;
                    case Beginning:
                    case End:
                    case Default:
                        client.subscribeToTopics(pattern, startPosition);
                        break;
                    default:
                        throw new KafkaClientInitializationException("Illegal 'startPosition' value for subscription with pattern: " + startPosition);
                }
            }
        }
        if (crContext != null && context.getPE().getRelaunchCount() > 0) {
            resettingLatch = new CountDownLatch(1);
        }
        processThread = getOperatorContext().getThreadFactory().newThread(new Runnable() {

            @Override
            public void run() {
                try {
                    processThreadEndedLatch = new CountDownLatch(1);
                    // initiates start polling if assigned or subscribed by sending an event
                    produceTuples();
                } catch (Exception e) {
                    // $NON-NLS-1$
                    Logger.getLogger(this.getClass()).error("Operator error", e);
                    // Otherwise this thread terminates leaving the PE in a healthy state without being healthy.
                    throw new RuntimeException(e.getLocalizedMessage(), e);
                } finally {
                    if (processThreadEndedLatch != null)
                        processThreadEndedLatch.countDown();
                    logger.info("process thread (tid = " + Thread.currentThread().getId() + ") ended.");
                }
            }
        });
        processThread.setDaemon(false);
    }
}

Example 9 with ConsumerClient

use of com.ibm.streamsx.kafka.clients.consumer.ConsumerClient in project streamsx.kafka by IBMStreams.

the class AbstractKafkaConsumerOperator method retireCheckpoint.

/**
 * @see com.ibm.streamsx.kafka.operators.AbstractKafkaOperator#retireCheckpoint(long)
 */
@Override
public void retireCheckpoint(long id) throws Exception {
    logger.debug(">>> RETIRE CHECKPOINT (ckpt id=" + id + ")");
    final ConsumerClient consumer = consumerRef.get();
    if (consumer.isProcessing()) {
        consumer.onCheckpointRetire(id);
    }
}

Example 10 with ConsumerClient

use of com.ibm.streamsx.kafka.clients.consumer.ConsumerClient in project streamsx.kafka by IBMStreams.

the class AbstractKafkaConsumerOperator method resetToInitialState.

@Override
public void resetToInitialState() throws Exception {
    final int attempt = crContext == null ? -1 : crContext.getResetAttempt();
    logger.log(DEBUG_LEVEL, MsgFormatter.format(">>> RESET TO INIT (attempt={0})", attempt));
    final long before = System.currentTimeMillis();
    final ConsumerClient consumer = consumerRef.get();
    if (consumer.isProcessing()) {
        // it is up to the consumer client implementation to stop polling.
        consumer.onResetToInitialState();
    }
    // by another PE, i.e. when relaunch count == 0 in initialize(context)
    if (resettingLatch != null)
        resettingLatch.countDown();
    final long after = System.currentTimeMillis();
    final long duration = after - before;
    logger.log(DEBUG_LEVEL, MsgFormatter.format(">>> RESET TO INIT took {0,number,#} ms (attempt={1})", duration, attempt));
}