Search in sources :

Example 1 with QuerySourceType

use of io.confluent.ksql.util.KsqlConstants.QuerySourceType in project ksql by confluentinc.

the class QueryMetricsUtil method initializePullTableMetricsCallback.

public static MetricsCallback initializePullTableMetricsCallback(final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final SlidingWindowRateLimiter pullBandRateLimiter, final AtomicReference<PullQueryResult> resultForMetrics) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> pullQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final PullQueryResult r = resultForMetrics.get();
        if (r == null) {
            recordErrorMetrics(pullQueryMetrics, responseBytes, startTimeNanos);
        } else {
            final QuerySourceType sourceType = r.getSourceType();
            final PullPhysicalPlanType planType = r.getPlanType();
            final RoutingNodeType routingNodeType = RoutingNodeType.SOURCE_NODE;
            metrics.recordResponseSize(responseBytes, sourceType, planType, routingNodeType);
            metrics.recordLatency(startTimeNanos, sourceType, planType, routingNodeType);
            metrics.recordRowsReturned(r.getTotalRowsReturned(), sourceType, planType, routingNodeType);
            metrics.recordRowsProcessed(r.getTotalRowsProcessed(), sourceType, planType, routingNodeType);
        }
        pullBandRateLimiter.add(responseBytes);
    });
    return metricsCallback;
}
Also used : ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) Decrementer(io.confluent.ksql.rest.util.ConcurrencyLimiter.Decrementer) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) QuerySourceType(io.confluent.ksql.util.KsqlConstants.QuerySourceType) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) MetricsCallback(io.confluent.ksql.api.server.MetricsCallback) State(org.apache.kafka.streams.KafkaStreams.State) AtomicReference(java.util.concurrent.atomic.AtomicReference) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) StreamPullQueryMetadata(io.confluent.ksql.util.StreamPullQueryMetadata) Optional(java.util.Optional) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) KafkaStreams(org.apache.kafka.streams.KafkaStreams) PullPhysicalPlanType(io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType) PullQueryResult(io.confluent.ksql.physical.pull.PullQueryResult) SlidingWindowRateLimiter(io.confluent.ksql.api.server.SlidingWindowRateLimiter) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) PullPhysicalPlanType(io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType) MetricsCallback(io.confluent.ksql.api.server.MetricsCallback) PullQueryResult(io.confluent.ksql.physical.pull.PullQueryResult) QuerySourceType(io.confluent.ksql.util.KsqlConstants.QuerySourceType)

Example 2 with QuerySourceType

use of io.confluent.ksql.util.KsqlConstants.QuerySourceType in project ksql by confluentinc.

the class QueryMetricsUtil method initializePullStreamMetricsCallback.

public static MetricsCallback initializePullStreamMetricsCallback(final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final SlidingWindowRateLimiter pullBandRateLimiter, final ImmutableAnalysis analysis, final AtomicReference<StreamPullQueryMetadata> resultForMetrics, final AtomicReference<Decrementer> refDecrementer) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> pullQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final StreamPullQueryMetadata m = resultForMetrics.get();
        final KafkaStreams.State state = m == null ? null : m.getTransientQueryMetadata().getKafkaStreams().state();
        if (m == null || state == null || state.equals(State.ERROR) || state.equals(State.PENDING_ERROR)) {
            recordErrorMetrics(pullQueryMetrics, responseBytes, startTimeNanos);
        } else {
            final boolean isWindowed = analysis.getFrom().getDataSource().getKsqlTopic().getKeyFormat().isWindowed();
            final QuerySourceType sourceType = isWindowed ? QuerySourceType.WINDOWED_STREAM : QuerySourceType.NON_WINDOWED_STREAM;
            // There is no WHERE clause constraint information in the persistent logical plan
            final PullPhysicalPlanType planType = PullPhysicalPlanType.UNKNOWN;
            final RoutingNodeType routingNodeType = RoutingNodeType.SOURCE_NODE;
            metrics.recordResponseSize(responseBytes, sourceType, planType, routingNodeType);
            metrics.recordLatency(startTimeNanos, sourceType, planType, routingNodeType);
            final TransientQueryQueue rowQueue = (TransientQueryQueue) m.getTransientQueryMetadata().getRowQueue();
            // The rows read from the underlying data source equal the rows read by the user
            // since the WHERE condition is pushed to the data source
            metrics.recordRowsReturned(rowQueue.getTotalRowsQueued(), sourceType, planType, routingNodeType);
            metrics.recordRowsProcessed(rowQueue.getTotalRowsQueued(), sourceType, planType, routingNodeType);
        }
        pullBandRateLimiter.add(responseBytes);
        // Decrement on happy or exception path
        final Decrementer decrementer = refDecrementer.get();
        if (decrementer != null) {
            decrementer.decrementAtMostOnce();
        }
    });
    return metricsCallback;
}
Also used : ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) Decrementer(io.confluent.ksql.rest.util.ConcurrencyLimiter.Decrementer) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) QuerySourceType(io.confluent.ksql.util.KsqlConstants.QuerySourceType) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) MetricsCallback(io.confluent.ksql.api.server.MetricsCallback) State(org.apache.kafka.streams.KafkaStreams.State) AtomicReference(java.util.concurrent.atomic.AtomicReference) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) StreamPullQueryMetadata(io.confluent.ksql.util.StreamPullQueryMetadata) Optional(java.util.Optional) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) KafkaStreams(org.apache.kafka.streams.KafkaStreams) PullPhysicalPlanType(io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType) PullQueryResult(io.confluent.ksql.physical.pull.PullQueryResult) SlidingWindowRateLimiter(io.confluent.ksql.api.server.SlidingWindowRateLimiter) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) KafkaStreams(org.apache.kafka.streams.KafkaStreams) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) State(org.apache.kafka.streams.KafkaStreams.State) PullPhysicalPlanType(io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType) StreamPullQueryMetadata(io.confluent.ksql.util.StreamPullQueryMetadata) MetricsCallback(io.confluent.ksql.api.server.MetricsCallback) Decrementer(io.confluent.ksql.rest.util.ConcurrencyLimiter.Decrementer) QuerySourceType(io.confluent.ksql.util.KsqlConstants.QuerySourceType)

Example 3 with QuerySourceType

use of io.confluent.ksql.util.KsqlConstants.QuerySourceType in project ksql by confluentinc.

the class QueryMetricsUtil method initializeScalablePushMetricsCallback.

public static MetricsCallback initializeScalablePushMetricsCallback(final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics, final SlidingWindowRateLimiter scalablePushBandRateLimiter, final AtomicReference<ScalablePushQueryMetadata> resultForMetrics) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> scalablePushQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final ScalablePushQueryMetadata r = resultForMetrics.get();
        if (r == null) {
            metrics.recordResponseSizeForError(responseBytes);
            metrics.recordConnectionDurationForError(startTimeNanos);
            metrics.recordZeroRowsReturnedForError();
            metrics.recordZeroRowsProcessedForError();
        } else {
            final QuerySourceType sourceType = r.getSourceType();
            final RoutingNodeType routingNodeType = r.getRoutingNodeType();
            metrics.recordResponseSize(responseBytes, sourceType, routingNodeType);
            metrics.recordConnectionDuration(startTimeNanos, sourceType, routingNodeType);
            metrics.recordRowsReturned(r.getTotalRowsReturned(), sourceType, routingNodeType);
            metrics.recordRowsProcessed(r.getTotalRowsProcessed(), sourceType, routingNodeType);
        }
        scalablePushBandRateLimiter.add(responseBytes);
    });
    return metricsCallback;
}
Also used : ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) Decrementer(io.confluent.ksql.rest.util.ConcurrencyLimiter.Decrementer) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) QuerySourceType(io.confluent.ksql.util.KsqlConstants.QuerySourceType) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) MetricsCallback(io.confluent.ksql.api.server.MetricsCallback) State(org.apache.kafka.streams.KafkaStreams.State) AtomicReference(java.util.concurrent.atomic.AtomicReference) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) StreamPullQueryMetadata(io.confluent.ksql.util.StreamPullQueryMetadata) Optional(java.util.Optional) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) KafkaStreams(org.apache.kafka.streams.KafkaStreams) PullPhysicalPlanType(io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType) PullQueryResult(io.confluent.ksql.physical.pull.PullQueryResult) SlidingWindowRateLimiter(io.confluent.ksql.api.server.SlidingWindowRateLimiter) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) MetricsCallback(io.confluent.ksql.api.server.MetricsCallback) QuerySourceType(io.confluent.ksql.util.KsqlConstants.QuerySourceType)

Example 4 with QuerySourceType

use of io.confluent.ksql.util.KsqlConstants.QuerySourceType in project ksql by confluentinc.

the class PullQueryExecutorMetrics method configureSensorMap.

private Map<MetricsKey, Sensor> configureSensorMap(final String sensorBaseName, final MetricsAdder metricsAdder) {
    final ImmutableMap.Builder<MetricsKey, Sensor> builder = ImmutableMap.builder();
    for (final QuerySourceType sourceType : QuerySourceType.values()) {
        for (final PullPhysicalPlanType planType : PullPhysicalPlanType.values()) {
            for (final RoutingNodeType routingNodeType : RoutingNodeType.values()) {
                addSensorToMap(sensorBaseName, metricsAdder, builder, new MetricsKey(sourceType, planType, routingNodeType));
            }
        }
    }
    // Add one more sensor for collecting metrics when there is no response
    addSensorToMap(sensorBaseName, metricsAdder, builder, new MetricsKey());
    return builder.build();
}
Also used : RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) PullPhysicalPlanType(io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType) ImmutableMap(com.google.common.collect.ImmutableMap) Sensor(org.apache.kafka.common.metrics.Sensor) QuerySourceType(io.confluent.ksql.util.KsqlConstants.QuerySourceType)

Example 5 with QuerySourceType

use of io.confluent.ksql.util.KsqlConstants.QuerySourceType in project ksql by confluentinc.

the class ScalablePushQueryMetrics method configureSensorMap.

private Map<MetricsKey, Sensor> configureSensorMap(final String sensorBaseName, final MetricsAdder metricsAdder) {
    final ImmutableMap.Builder<MetricsKey, Sensor> builder = ImmutableMap.builder();
    for (final QuerySourceType sourceType : QuerySourceType.values()) {
        for (final RoutingNodeType routingNodeType : RoutingNodeType.values()) {
            addSensorToMap(sensorBaseName, metricsAdder, builder, new MetricsKey(sourceType, routingNodeType));
        }
    }
    // Add one more sensor for collecting metrics when there is no response
    addSensorToMap(sensorBaseName, metricsAdder, builder, new MetricsKey());
    return builder.build();
}
Also used : RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) ImmutableMap(com.google.common.collect.ImmutableMap) Sensor(org.apache.kafka.common.metrics.Sensor) QuerySourceType(io.confluent.ksql.util.KsqlConstants.QuerySourceType)

Aggregations

QuerySourceType (io.confluent.ksql.util.KsqlConstants.QuerySourceType)5 RoutingNodeType (io.confluent.ksql.util.KsqlConstants.RoutingNodeType)5 PullPhysicalPlanType (io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType)4 ImmutableAnalysis (io.confluent.ksql.analyzer.ImmutableAnalysis)3 MetricsCallback (io.confluent.ksql.api.server.MetricsCallback)3 SlidingWindowRateLimiter (io.confluent.ksql.api.server.SlidingWindowRateLimiter)3 PullQueryExecutorMetrics (io.confluent.ksql.internal.PullQueryExecutorMetrics)3 ScalablePushQueryMetrics (io.confluent.ksql.internal.ScalablePushQueryMetrics)3 PullQueryResult (io.confluent.ksql.physical.pull.PullQueryResult)3 TransientQueryQueue (io.confluent.ksql.query.TransientQueryQueue)3 Decrementer (io.confluent.ksql.rest.util.ConcurrencyLimiter.Decrementer)3 ScalablePushQueryMetadata (io.confluent.ksql.util.ScalablePushQueryMetadata)3 StreamPullQueryMetadata (io.confluent.ksql.util.StreamPullQueryMetadata)3 Optional (java.util.Optional)3 AtomicReference (java.util.concurrent.atomic.AtomicReference)3 KafkaStreams (org.apache.kafka.streams.KafkaStreams)3 State (org.apache.kafka.streams.KafkaStreams.State)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 Sensor (org.apache.kafka.common.metrics.Sensor)2