Examples with MetricsFetcher com.ibm.streamsx.kafka.clients.metrics.MetricsFetcher used on opensource projects

Example 1 with MetricsFetcher

use of com.ibm.streamsx.kafka.clients.metrics.MetricsFetcher in project streamsx.kafka by IBMStreams.

the class AbstractKafkaProducerClient method createProducer.

protected final synchronized void createProducer() {
    final String applicationDirectory = getOperatorContext().getPE().getApplicationDirectory().getAbsolutePath();
    SystemProperties.resolveApplicationDir(applicationDirectory);
    producer = new KafkaProducer<>(this.kafkaProperties);
    producerGenerationMetric.increment();
    if (metricsFetcher == null) {
        metricsFetcher = new MetricsFetcher(getOperatorContext(), new MetricsProvider() {

            @Override
            public Map<MetricName, ? extends Metric> getMetrics() {
                synchronized (AbstractKafkaProducerClient.this) {
                    return producer.metrics();
                }
            }

            @Override
            public String createCustomMetricName(MetricName metricName) throws KafkaMetricException {
                return ProducerMetricsReporter.createOperatorMetricName(metricName);
            }
        }, ProducerMetricsReporter.getMetricsFilter(), AbstractKafkaClient.METRICS_REPORT_INTERVAL);
        metricsFetcher.registerUpdateListener(this.metricsMonitor);
        metricsFetcher.registerUpdateListener("record-queue-time-max", new CustomMetricUpdateListener() {

            @Override
            public void customMetricUpdated(final String customMetricName, final MetricName kafkaMetricName, final long value) {
                metricsMonitor.setRecordQueueTimeMax(value);
                recordQueueTimeMaxMName.compareAndSet(null, kafkaMetricName);
            }
        });
        metricsFetcher.registerUpdateListener("outgoing-byte-rate", new CustomMetricUpdateListener() {

            @Override
            public void customMetricUpdated(final String customMetricName, final MetricName kafkaMetricName, final long value) {
                metricsMonitor.setOutgoingByteRate(value);
                outGoingByteRateMName.compareAndSet(null, kafkaMetricName);
            }
        });
        metricsFetcher.registerUpdateListener("records-per-request-avg", new CustomMetricUpdateListener() {

            @Override
            public void customMetricUpdated(final String customMetricName, final MetricName kafkaMetricName, final long value) {
                metricsMonitor.setRecordsPerRequestAvg(value);
            }
        });
        metricsFetcher.registerUpdateListener("batch-size-avg", new CustomMetricUpdateListener() {

            @Override
            public void customMetricUpdated(final String customMetricName, final MetricName kafkaMetricName, final long value) {
                metricsMonitor.setBatchSizeAvg(value);
            }
        });
        metricsFetcher.registerUpdateListener("buffer-available-bytes", new CustomMetricUpdateListener() {

            @Override
            public void customMetricUpdated(final String customMetricName, final MetricName kafkaMetricName, final long value) {
                metricsMonitor.setBufferAvailBytes(value);
                bufferAvailMName.compareAndSet(null, kafkaMetricName);
            }
        });
        metricsFetcher.registerUpdateListener("record-queue-time-avg", new CustomMetricUpdateListener() {

            @Override
            public void customMetricUpdated(final String customMetricName, final MetricName kafkaMetricName, final long value) {
                metricsMonitor.setRecordQueueTimeAvg(value);
            }
        });
        metricsFetcher.registerUpdateListener("bufferpool-wait-time-total", new CustomMetricUpdateListener() {

            @Override
            public void customMetricUpdated(final String customMetricName, final MetricName kafkaMetricName, final long value) {
                metricsMonitor.setBufferPoolWaitTimeTotalNanos(value);
            }
        });
        metricsFetcher.registerUpdateListener("request-rate", new CustomMetricUpdateListener() {

            @Override
            public void customMetricUpdated(final String customMetricName, final MetricName kafkaMetricName, final long value) {
                metricsMonitor.setRequestRate(value);
            }
        });
    }
}

Example 2 with MetricsFetcher

use of com.ibm.streamsx.kafka.clients.metrics.MetricsFetcher in project streamsx.kafka by IBMStreams.

the class AbstractKafkaConsumerClient method startConsumer.

/**
 * Validates the setup of the consumer client by calling the {@link #validate()} method,
 * creates the Kafka consumer object and starts the consumer and event thread.
 * This method ensures that the event thread is running when it returns.
 * Methods that overwrite this method must call super.startConsumer().
 * @throws InterruptedException The thread has been interrupted.
 * @throws KafkaClientInitializationException The client could not be initialized
 */
public void startConsumer() throws InterruptedException, KafkaClientInitializationException {
    try {
        validate();
    } catch (Exception e) {
        throw new KafkaClientInitializationException(e.getLocalizedMessage(), e);
    }
    consumerInitLatch = new CountDownLatch(1);
    Thread eventThread = getOperatorContext().getThreadFactory().newThread(new Runnable() {

        @Override
        public void run() {
            try {
                maxPollRecords = getMaxPollRecordsFromProperties(kafkaProperties);
                maxPollIntervalMs = getMaxPollIntervalMsFromProperties(kafkaProperties);
                fetchMaxBytes = getFetchMaxBytesFromProperties(kafkaProperties);
                SystemProperties.resolveApplicationDir(getOperatorContext().getPE().getApplicationDirectory().getAbsolutePath());
                consumer = new KafkaConsumer<>(kafkaProperties);
                processing.set(true);
            } catch (Exception e) {
                initializationException = e;
                return;
            } finally {
                // notify that consumer is ready
                consumerInitLatch.countDown();
            }
            try {
                runEventLoop();
            } catch (InterruptedException e) {
                logger.debug("Event thread interrupted. Terminating thread.");
                return;
            } finally {
                processing.set(false);
                logger.info("event thread (tid = " + Thread.currentThread().getId() + ") ended.");
            }
        }
    });
    eventThread.setDaemon(false);
    eventThread.start();
    // wait for consumer thread to be running before returning
    consumerInitLatch.await();
    if (this.metricsFetcher == null) {
        this.metricsFetcher = new MetricsFetcher(getOperatorContext(), new MetricsProvider() {

            @Override
            public Map<MetricName, ? extends org.apache.kafka.common.Metric> getMetrics() {
                return consumer.metrics();
            }

            @Override
            public String createCustomMetricName(MetricName metricName) throws KafkaMetricException {
                return ConsumerMetricsReporter.createOperatorMetricName(metricName);
            }
        }, ConsumerMetricsReporter.getMetricsFilter(), AbstractKafkaClient.METRICS_REPORT_INTERVAL);
    }
    if (initializationException != null)
        throw new KafkaClientInitializationException(initializationException.getLocalizedMessage(), initializationException);
}