Examples with ProcessingElement com.ibm.streams.operator.ProcessingElement used on opensource projects

Example 1 with ProcessingElement

use of com.ibm.streams.operator.ProcessingElement in project streamsx.kafka by IBMStreams.

the class AbstractKafkaConsumerClient method generateGroupId.

/**
 * Generates a group identifier that is consistent accross PE relaunches, but not accross job submissions.
 * @param operatorContext
 * @return a group identifier
 */
private String generateGroupId(final OperatorContext context) {
    final ProcessingElement pe = context.getPE();
    final int iidH = pe.getInstanceId().hashCode();
    final int opnH = context.getName().hashCode();
    final String id = MsgFormatter.format("i{0}-j{1}-o{2}", (iidH < 0 ? "N" + (-iidH) : "P" + iidH), "" + pe.getJobId(), (opnH < 0 ? "N" + (-opnH) : "P" + opnH));
    return id;
}

Example 2 with ProcessingElement

use of com.ibm.streams.operator.ProcessingElement in project streamsx.kafka by IBMStreams.

the class AbstractKafkaConsumerOperator method initialize.

@Override
public void initialize(OperatorContext context) throws Exception {
    synchronized (monitor) {
        // Must call super.initialize(context) to correctly setup an operator.
        super.initialize(context);
        logger.info(// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
        "Operator " + context.getName() + " initializing in PE: " + context.getPE().getPEId() + " in Job: " + context.getPE().getJobId());
        shutdown = new AtomicBoolean(false);
        StreamSchema outputSchema = context.getStreamingOutputs().get(0).getStreamSchema();
        outputMessageAttrIdx = outputSchema.getAttributeIndex(outputMessageAttrName);
        outputKeyAttrIdx = outputSchema.getAttributeIndex(outputKeyAttrName);
        outputTopicAttrIdx = outputSchema.getAttributeIndex(outputTopicAttrName);
        outputTimetampAttrIdx = outputSchema.getAttributeIndex(outputMessageTimestampAttrName);
        outputPartitionAttrIdx = outputSchema.getAttributeIndex(outputPartitionAttrName);
        outputOffsetAttrIdx = outputSchema.getAttributeIndex(outputOffsetAttrName);
        Class<?> keyClass = outputKeyAttrIdx >= 0 ? getAttributeType(context.getStreamingOutputs().get(0), outputKeyAttrName) : // default to String.class for key type
        String.class;
        Class<?> valueClass = getAttributeType(context.getStreamingOutputs().get(0), outputMessageAttrName);
        KafkaOperatorProperties kafkaProperties = getKafkaProperties();
        if (this.startPosition == StartPosition.Time && !context.getParameterNames().contains(START_TIME_PARAM)) {
            throw new KafkaConfigurationException(Messages.getString("START_TIME_PARAM_NOT_FOUND"));
        }
        if (this.startPosition == StartPosition.Offset && !context.getParameterNames().contains(START_OFFSET_PARAM)) {
            throw new KafkaConfigurationException(Messages.getString("START_OFFSET_PARAM_NOT_FOUND"));
        }
        // set the group ID property if the groupId parameter is specified
        if (groupId != null && !groupId.isEmpty()) {
            kafkaProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId);
        }
        final boolean hasInputPorts = context.getStreamingInputs().size() > 0;
        final String gid = kafkaProperties.getProperty(ConsumerConfig.GROUP_ID_CONFIG);
        this.groupIdSpecified = gid != null && !gid.trim().isEmpty();
        logger.log(DEBUG_LEVEL, "group-ID specified: " + this.groupIdSpecified);
        if (crContext != null) {
            commitMode = CommitMode.ConsistentRegionDrain;
        } else {
            final Set<String> parameterNames = context.getParameterNames();
            commitMode = parameterNames.contains(COMMIT_COUNT_PARAM) ? CommitMode.TupleCount : CommitMode.Time;
        }
        if (this.staticGroupMember) {
            // calculate a unique group.instance.id that is consistent across operator restarts
            final ProcessingElement pe = context.getPE();
            final int iidH = pe.getInstanceId().hashCode();
            final int opnH = context.getName().hashCode();
            final String groupInstanceId = MsgFormatter.format("i{0}-o{1}", (iidH < 0 ? "N" + (-iidH) : "P" + iidH), (opnH < 0 ? "N" + (-opnH) : "P" + opnH));
            logger.info("Generated group.instance.id: " + groupInstanceId);
            kafkaProperties.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG, groupInstanceId);
        }
        // create the builders for the consumer clients
        if (crContext == null) {
            this.groupEnabledClientBuilder = new NonCrKafkaConsumerGroupClient.Builder().setOperatorContext(context).setKafkaProperties(kafkaProperties).setKeyClass(keyClass).setValueClass(valueClass).setSingleTopic(this.topics != null && this.topics.size() == 1).setPollTimeout(this.consumerPollTimeout).setInitialStartPosition(this.startPosition).setCommitMode(commitMode).setCommitPeriod(commitPeriod).setCommitCount(commitCount);
            this.staticAssignClientBuilder = new NonCrKafkaConsumerClient.Builder().setOperatorContext(context).setKafkaProperties(kafkaProperties).setKeyClass(keyClass).setValueClass(valueClass).setPollTimeout(this.consumerPollTimeout).setInitialStartPosition(this.startPosition).setCommitMode(commitMode).setCommitPeriod(commitPeriod).setCommitCount(commitCount);
        } else {
            // CR
            this.groupEnabledClientBuilder = new CrKafkaConsumerGroupClient.Builder().setOperatorContext(context).setKafkaProperties(kafkaProperties).setKeyClass(keyClass).setValueClass(valueClass).setPollTimeout(this.consumerPollTimeout).setSingleTopic(this.topics != null && this.topics.size() == 1).setTriggerCount(this.triggerCount).setInitialStartPosition(this.startPosition).setInitialStartTimestamp(this.startTime);
            this.staticAssignClientBuilder = new CrKafkaStaticAssignConsumerClient.Builder().setOperatorContext(context).setKafkaProperties(kafkaProperties).setKeyClass(keyClass).setValueClass(valueClass).setPollTimeout(this.consumerPollTimeout).setTriggerCount(this.triggerCount);
        }
        magics.put(this.staticAssignClientBuilder.getImplementationMagic(), this.staticAssignClientBuilder);
        magics.put(this.groupEnabledClientBuilder.getImplementationMagic(), this.groupEnabledClientBuilder);
        final ConsumerClientBuilder builder;
        if (hasInputPorts) {
            if (crContext != null) {
                // in CR, we do not groupManagement with input port:
                builder = this.staticAssignClientBuilder;
            } else {
                // not in CR: select the right builder in checkpoint reset or on first partition/topic addition
                builder = new DummyConsumerClient.Builder().setOperatorContext(context).setKafkaProperties(kafkaProperties);
                magics.put(builder.getImplementationMagic(), builder);
            }
        } else {
            boolean groupManagementEnabled;
            if (Features.ENABLE_GROUP_MANAGEMENT_NO_USER_GROUP_ID) {
                groupManagementEnabled = this.partitions == null || this.partitions.isEmpty();
            } else {
                // legacy (2.x) behavior
                groupManagementEnabled = this.groupIdSpecified && (this.partitions == null || this.partitions.isEmpty());
            }
            if (this.groupIdSpecified && !groupManagementEnabled) {
                if (this.partitions != null && !this.partitions.isEmpty()) {
                    logger.warn(MsgFormatter.format("The group.id ''{0}'' is specified. The ''{1}'' operator " + "will NOT participate in a consumer group because partitions to consume are specified.", gid, context.getName()));
                }
            }
            // when group management is disabled and no input port is configured, we must not subscribe with pattern
            // When we are here it is already guaranteed that we have one of the 'topic' or 'pattern' parameter
            final boolean p = this.pattern != null;
            final boolean t = this.topics != null;
            assert ((p && !t) || (t && !p));
            if (!groupManagementEnabled && p) {
                final String msg = Messages.getString("PATTERN_SUBSCRIPTION_REQUIRES_GROUP_MGT", PATTERN_PARAM, context.getName(), context.getKind());
                logger.error(msg);
                throw new KafkaConfigurationException(msg);
            }
            builder = groupManagementEnabled ? this.groupEnabledClientBuilder : this.staticAssignClientBuilder;
        }
        ConsumerClient client = builder.build();
        consumerRef = new AtomicReference<>(client);
        logger.info(MsgFormatter.format("consumer client {0} created", client.getClass().getName()));
        try {
            client.startConsumer();
        } catch (KafkaClientInitializationException e) {
            e.printStackTrace();
            logger.error(e.getLocalizedMessage(), e);
            logger.error("root cause: " + e.getRootCause());
            throw e;
        }
        // input port not used, so topic or pattern must be defined
        if (!hasInputPorts) {
            if (this.topics != null) {
                final boolean registerAsInput = true;
                registerForDataGovernance(context, topics, registerAsInput);
                switch(startPosition) {
                    case Time:
                        client.subscribeToTopicsWithTimestamp(topics, partitions, startTime);
                        break;
                    case Offset:
                        client.subscribeToTopicsWithOffsets(topics.get(0), partitions, startOffsets);
                        break;
                    default:
                        client.subscribeToTopics(topics, partitions, startPosition);
                }
            } else {
                switch(startPosition) {
                    case Time:
                        client.subscribeToTopicsWithTimestamp(pattern, startTime);
                        break;
                    case Beginning:
                    case End:
                    case Default:
                        client.subscribeToTopics(pattern, startPosition);
                        break;
                    default:
                        throw new KafkaClientInitializationException("Illegal 'startPosition' value for subscription with pattern: " + startPosition);
                }
            }
        }
        if (crContext != null && context.getPE().getRelaunchCount() > 0) {
            resettingLatch = new CountDownLatch(1);
        }
        processThread = getOperatorContext().getThreadFactory().newThread(new Runnable() {

            @Override
            public void run() {
                try {
                    processThreadEndedLatch = new CountDownLatch(1);
                    // initiates start polling if assigned or subscribed by sending an event
                    produceTuples();
                } catch (Exception e) {
                    // $NON-NLS-1$
                    Logger.getLogger(this.getClass()).error("Operator error", e);
                    // Otherwise this thread terminates leaving the PE in a healthy state without being healthy.
                    throw new RuntimeException(e.getLocalizedMessage(), e);
                } finally {
                    if (processThreadEndedLatch != null)
                        processThreadEndedLatch.countDown();
                    logger.info("process thread (tid = " + Thread.currentThread().getId() + ") ended.");
                }
            }
        });
        processThread.setDaemon(false);
    }
}