Examples with QuerySourceType io.confluent.ksql.util.KsqlConstants.QuerySourceType used on opensource projects

Example 1 with QuerySourceType

use of io.confluent.ksql.util.KsqlConstants.QuerySourceType in project ksql by confluentinc.

the class QueryMetricsUtil method initializePullTableMetricsCallback.

public static MetricsCallback initializePullTableMetricsCallback(final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final SlidingWindowRateLimiter pullBandRateLimiter, final AtomicReference<PullQueryResult> resultForMetrics) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> pullQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final PullQueryResult r = resultForMetrics.get();
        if (r == null) {
            recordErrorMetrics(pullQueryMetrics, responseBytes, startTimeNanos);
        } else {
            final QuerySourceType sourceType = r.getSourceType();
            final PullPhysicalPlanType planType = r.getPlanType();
            final RoutingNodeType routingNodeType = RoutingNodeType.SOURCE_NODE;
            metrics.recordResponseSize(responseBytes, sourceType, planType, routingNodeType);
            metrics.recordLatency(startTimeNanos, sourceType, planType, routingNodeType);
            metrics.recordRowsReturned(r.getTotalRowsReturned(), sourceType, planType, routingNodeType);
            metrics.recordRowsProcessed(r.getTotalRowsProcessed(), sourceType, planType, routingNodeType);
        }
        pullBandRateLimiter.add(responseBytes);
    });
    return metricsCallback;
}

Example 2 with QuerySourceType

use of io.confluent.ksql.util.KsqlConstants.QuerySourceType in project ksql by confluentinc.

the class QueryMetricsUtil method initializePullStreamMetricsCallback.

public static MetricsCallback initializePullStreamMetricsCallback(final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final SlidingWindowRateLimiter pullBandRateLimiter, final ImmutableAnalysis analysis, final AtomicReference<StreamPullQueryMetadata> resultForMetrics, final AtomicReference<Decrementer> refDecrementer) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> pullQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final StreamPullQueryMetadata m = resultForMetrics.get();
        final KafkaStreams.State state = m == null ? null : m.getTransientQueryMetadata().getKafkaStreams().state();
        if (m == null || state == null || state.equals(State.ERROR) || state.equals(State.PENDING_ERROR)) {
            recordErrorMetrics(pullQueryMetrics, responseBytes, startTimeNanos);
        } else {
            final boolean isWindowed = analysis.getFrom().getDataSource().getKsqlTopic().getKeyFormat().isWindowed();
            final QuerySourceType sourceType = isWindowed ? QuerySourceType.WINDOWED_STREAM : QuerySourceType.NON_WINDOWED_STREAM;
            // There is no WHERE clause constraint information in the persistent logical plan
            final PullPhysicalPlanType planType = PullPhysicalPlanType.UNKNOWN;
            final RoutingNodeType routingNodeType = RoutingNodeType.SOURCE_NODE;
            metrics.recordResponseSize(responseBytes, sourceType, planType, routingNodeType);
            metrics.recordLatency(startTimeNanos, sourceType, planType, routingNodeType);
            final TransientQueryQueue rowQueue = (TransientQueryQueue) m.getTransientQueryMetadata().getRowQueue();
            // The rows read from the underlying data source equal the rows read by the user
            // since the WHERE condition is pushed to the data source
            metrics.recordRowsReturned(rowQueue.getTotalRowsQueued(), sourceType, planType, routingNodeType);
            metrics.recordRowsProcessed(rowQueue.getTotalRowsQueued(), sourceType, planType, routingNodeType);
        }
        pullBandRateLimiter.add(responseBytes);
        // Decrement on happy or exception path
        final Decrementer decrementer = refDecrementer.get();
        if (decrementer != null) {
            decrementer.decrementAtMostOnce();
        }
    });
    return metricsCallback;
}

Example 3 with QuerySourceType

use of io.confluent.ksql.util.KsqlConstants.QuerySourceType in project ksql by confluentinc.

the class QueryMetricsUtil method initializeScalablePushMetricsCallback.

public static MetricsCallback initializeScalablePushMetricsCallback(final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics, final SlidingWindowRateLimiter scalablePushBandRateLimiter, final AtomicReference<ScalablePushQueryMetadata> resultForMetrics) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> scalablePushQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final ScalablePushQueryMetadata r = resultForMetrics.get();
        if (r == null) {
            metrics.recordResponseSizeForError(responseBytes);
            metrics.recordConnectionDurationForError(startTimeNanos);
            metrics.recordZeroRowsReturnedForError();
            metrics.recordZeroRowsProcessedForError();
        } else {
            final QuerySourceType sourceType = r.getSourceType();
            final RoutingNodeType routingNodeType = r.getRoutingNodeType();
            metrics.recordResponseSize(responseBytes, sourceType, routingNodeType);
            metrics.recordConnectionDuration(startTimeNanos, sourceType, routingNodeType);
            metrics.recordRowsReturned(r.getTotalRowsReturned(), sourceType, routingNodeType);
            metrics.recordRowsProcessed(r.getTotalRowsProcessed(), sourceType, routingNodeType);
        }
        scalablePushBandRateLimiter.add(responseBytes);
    });
    return metricsCallback;
}

Example 4 with QuerySourceType

use of io.confluent.ksql.util.KsqlConstants.QuerySourceType in project ksql by confluentinc.

the class PullQueryExecutorMetrics method configureSensorMap.

private Map<MetricsKey, Sensor> configureSensorMap(final String sensorBaseName, final MetricsAdder metricsAdder) {
    final ImmutableMap.Builder<MetricsKey, Sensor> builder = ImmutableMap.builder();
    for (final QuerySourceType sourceType : QuerySourceType.values()) {
        for (final PullPhysicalPlanType planType : PullPhysicalPlanType.values()) {
            for (final RoutingNodeType routingNodeType : RoutingNodeType.values()) {
                addSensorToMap(sensorBaseName, metricsAdder, builder, new MetricsKey(sourceType, planType, routingNodeType));
            }
        }
    }
    // Add one more sensor for collecting metrics when there is no response
    addSensorToMap(sensorBaseName, metricsAdder, builder, new MetricsKey());
    return builder.build();
}

Example 5 with QuerySourceType

use of io.confluent.ksql.util.KsqlConstants.QuerySourceType in project ksql by confluentinc.

the class ScalablePushQueryMetrics method configureSensorMap.

private Map<MetricsKey, Sensor> configureSensorMap(final String sensorBaseName, final MetricsAdder metricsAdder) {
    final ImmutableMap.Builder<MetricsKey, Sensor> builder = ImmutableMap.builder();
    for (final QuerySourceType sourceType : QuerySourceType.values()) {
        for (final RoutingNodeType routingNodeType : RoutingNodeType.values()) {
            addSensorToMap(sensorBaseName, metricsAdder, builder, new MetricsKey(sourceType, routingNodeType));
        }
    }
    // Add one more sensor for collecting metrics when there is no response
    addSensorToMap(sensorBaseName, metricsAdder, builder, new MetricsKey());
    return builder.build();
}