Examples with ControlPlaneContext com.ibm.streams.operator.control.ControlPlaneContext used on opensource projects

Example 1 with ControlPlaneContext

use of com.ibm.streams.operator.control.ControlPlaneContext in project streamsx.kafka by IBMStreams.

the class KafkaConsumerClient method saveOffsetManagerToJCP.

private void saveOffsetManagerToJCP() throws Exception {
    ControlPlaneContext controlPlaneContext = operatorContext.getOptionalContext(ControlPlaneContext.class);
    offsetManagerCV = controlPlaneContext.createStringControlVariable(OffsetManager.class.getName(), false, serializeObject(offsetManager));
    OffsetManager mgr = getDeserializedOffsetManagerCV();
    logger.debug("Retrieved value for offsetManagerCV=" + mgr);
}

Example 2 with ControlPlaneContext

use of com.ibm.streams.operator.control.ControlPlaneContext in project streamsx.kafka by IBMStreams.

the class TransactionalKafkaProducerClient method configureProperties.

@Override
protected void configureProperties() throws Exception {
    super.configureProperties();
    // Need to generate a transaction ID that is unique but persists
    // across operator instances. In order to guarantee this, we will
    // store the transaction ID in the JCP
    ControlPlaneContext crContext = operatorContext.getOptionalContext(ControlPlaneContext.class);
    ControlVariableAccessor<String> transactionalIdCV = crContext.createStringControlVariable("transactional_id", false, getRandomId("tid-"));
    transactionalId = transactionalIdCV.sync().getValue();
    logger.debug("Transactional ID = " + transactionalId);
    // The "enable.idempotence" property is required in order to guarantee idempotence
    this.kafkaProperties.setProperty(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true");
    // The "transactional.id" property is mandatory in order to support transactions.
    this.kafkaProperties.setProperty(ProducerConfig.TRANSACTIONAL_ID_CONFIG, transactionalId);
}

Example 3 with ControlPlaneContext

use of com.ibm.streams.operator.control.ControlPlaneContext in project streamsx.topology by IBMStreams.

the class Resetter method initialize.

@Override
public synchronized void initialize(OperatorContext context) throws Exception {
    super.initialize(context);
    createConditionMetrics();
    Metric mrm = getOperatorContext().getMetrics().createCustomMetric("nMinimumResets", "Minimum number of resets per channel", Metric.Kind.COUNTER);
    mrm.setValue(getMinimumResets());
    ControlPlaneContext control = context.getOptionalContext(ControlPlaneContext.class);
    control.connect(this);
}

Example 4 with ControlPlaneContext

use of com.ibm.streams.operator.control.ControlPlaneContext in project streamsx.kafka by IBMStreams.

the class TransactionalCrProducerClient method configureProperties.

/**
 */
@Override
protected void configureProperties() throws Exception {
    super.configureProperties();
    // Need to generate a transactional.id that is unique but persists
    // across operator re-launches. In order to guarantee this, we will
    // store the transactional.id in the JCP
    ControlPlaneContext jcpContext = getOperatorContext().getOptionalContext(ControlPlaneContext.class);
    ControlVariableAccessor<String> transactionalIdCV = jcpContext.createStringControlVariable("transactional_id", false, getRandomId("tid-"));
    transactionalId = transactionalIdCV.sync().getValue();
    trace.debug("Transactional ID = " + transactionalId);
    // adjust transaction timeout transaction.timeout.ms
    ConsistentRegionContext crContext = getOperatorContext().getOptionalContext(ConsistentRegionContext.class);
    long drainTimeoutMillis = (long) (crContext.getDrainTimeout() * 1000.0);
    long minTransactionTimeout = drainTimeoutMillis + 120000l;
    if (minTransactionTimeout > TRANSACTION_MAX_TIMEOUT_MS)
        minTransactionTimeout = TRANSACTION_MAX_TIMEOUT_MS;
    if (kafkaProperties.containsKey(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG)) {
        long propValue = Long.valueOf(kafkaProperties.getProperty(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG));
        if (propValue < minTransactionTimeout) {
            this.kafkaProperties.setProperty(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, "" + minTransactionTimeout);
            trace.warn(MsgFormatter.format("producer config ''{0}'' has been increased from {1} to {2}.", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, propValue, minTransactionTimeout));
        }
    } else {
        this.kafkaProperties.setProperty(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, "" + minTransactionTimeout);
    }
    // The "enable.idempotence" property is required in order to guarantee idempotence
    this.kafkaProperties.setProperty(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true");
    // to be less than or equal to 5, retries to be greater than 0 and acks must be 'all'.
    if (kafkaProperties.containsKey(ProducerConfig.ACKS_CONFIG)) {
        final String acks = kafkaProperties.getProperty(ProducerConfig.ACKS_CONFIG);
        if (!(acks.equals("all") || acks.equals("-1"))) {
            trace.warn(MsgFormatter.format("producer config ''{0}'' has been changed from {1} to {2} for enable.idempotence=true.", ProducerConfig.ACKS_CONFIG, acks, "all"));
            this.kafkaProperties.setProperty(ProducerConfig.ACKS_CONFIG, "all");
        }
    } else
        this.kafkaProperties.setProperty(ProducerConfig.ACKS_CONFIG, "all");
    if (kafkaProperties.containsKey(ProducerConfig.RETRIES_CONFIG)) {
        final long retries = Long.parseLong(kafkaProperties.getProperty(ProducerConfig.RETRIES_CONFIG).trim());
        if (retries < 1l) {
            trace.warn(MsgFormatter.format("producer config ''{0}'' has been changed from {1} to {2} for enable.idempotence=true.", ProducerConfig.RETRIES_CONFIG, retries, "1"));
            this.kafkaProperties.setProperty(ProducerConfig.RETRIES_CONFIG, "1");
        }
    }
    // eanble.idempotence requires max.in.flight.requests.per.connection <= 5
    final long maxInFlightRequestsForIdempotence = 5L;
    if (kafkaProperties.containsKey(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION)) {
        final long maxInFlightRequests = Long.parseLong(kafkaProperties.getProperty(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION).trim());
        if (maxInFlightRequests > maxInFlightRequestsForIdempotence) {
            final String val = "" + maxInFlightRequestsForIdempotence;
            trace.warn(MsgFormatter.format("producer config ''{0}'' has been reduced from {1} to {2} for enable.idempotence=true.", ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, maxInFlightRequests, val));
            this.kafkaProperties.setProperty(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, val);
        }
    } else {
        // config not set:
        this.kafkaProperties.setProperty(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "" + maxInFlightRequestsForIdempotence);
    }
    // --- end adjustment for enable.idempotence = true
    // The "transactional.id" property is mandatory in order to support transactions.
    this.kafkaProperties.setProperty(ProducerConfig.TRANSACTIONAL_ID_CONFIG, transactionalId);
}