Examples with TransientQueryQueue io.confluent.ksql.query.TransientQueryQueue used on opensource projects

Example 6 with TransientQueryQueue

use of io.confluent.ksql.query.TransientQueryQueue in project ksql by confluentinc.

the class PushRoutingTest method shouldFail_hitRequestLimitRemote.

@Test
public void shouldFail_hitRequestLimitRemote() throws ExecutionException, InterruptedException {
    // Given:
    when(locator.locate()).thenReturn(ImmutableList.of(ksqlNodeRemote));
    transientQueryQueue = new TransientQueryQueue(OptionalInt.empty(), 1, 100);
    final PushRouting routing = new PushRouting();
    // When:
    final PushConnectionsHandle handle = handlePushRouting(routing);
    context.runOnContext(v -> {
        remotePublisher.accept(REMOTE_ROW1);
        remotePublisher.accept(REMOTE_ROW2);
    });
    // Then:
    Set<List<?>> rows = waitOnRows(1);
    handle.close();
    assertThat(rows.contains(REMOTE_ROW1.getRow().get().getColumns()), is(true));
    assertThat(handle.getError().getMessage(), containsString("Hit limit of request queue"));
}

Example 7 with TransientQueryQueue

use of io.confluent.ksql.query.TransientQueryQueue in project ksql by confluentinc.

the class QueryMetricsUtil method initializePullStreamMetricsCallback.

public static MetricsCallback initializePullStreamMetricsCallback(final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final SlidingWindowRateLimiter pullBandRateLimiter, final ImmutableAnalysis analysis, final AtomicReference<StreamPullQueryMetadata> resultForMetrics, final AtomicReference<Decrementer> refDecrementer) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> pullQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final StreamPullQueryMetadata m = resultForMetrics.get();
        final KafkaStreams.State state = m == null ? null : m.getTransientQueryMetadata().getKafkaStreams().state();
        if (m == null || state == null || state.equals(State.ERROR) || state.equals(State.PENDING_ERROR)) {
            recordErrorMetrics(pullQueryMetrics, responseBytes, startTimeNanos);
        } else {
            final boolean isWindowed = analysis.getFrom().getDataSource().getKsqlTopic().getKeyFormat().isWindowed();
            final QuerySourceType sourceType = isWindowed ? QuerySourceType.WINDOWED_STREAM : QuerySourceType.NON_WINDOWED_STREAM;
            // There is no WHERE clause constraint information in the persistent logical plan
            final PullPhysicalPlanType planType = PullPhysicalPlanType.UNKNOWN;
            final RoutingNodeType routingNodeType = RoutingNodeType.SOURCE_NODE;
            metrics.recordResponseSize(responseBytes, sourceType, planType, routingNodeType);
            metrics.recordLatency(startTimeNanos, sourceType, planType, routingNodeType);
            final TransientQueryQueue rowQueue = (TransientQueryQueue) m.getTransientQueryMetadata().getRowQueue();
            // The rows read from the underlying data source equal the rows read by the user
            // since the WHERE condition is pushed to the data source
            metrics.recordRowsReturned(rowQueue.getTotalRowsQueued(), sourceType, planType, routingNodeType);
            metrics.recordRowsProcessed(rowQueue.getTotalRowsQueued(), sourceType, planType, routingNodeType);
        }
        pullBandRateLimiter.add(responseBytes);
        // Decrement on happy or exception path
        final Decrementer decrementer = refDecrementer.get();
        if (decrementer != null) {
            decrementer.decrementAtMostOnce();
        }
    });
    return metricsCallback;
}