Search in sources :

Example 1 with ScalablePushQueryMetrics

use of io.confluent.ksql.internal.ScalablePushQueryMetrics in project ksql by confluentinc.

the class PushRouting method connectToHosts.

/**
 * Connects to all of the hosts provided.
 * @return A future for a PushConnectionsHandle, which can be used to terminate connections.
 */
@SuppressWarnings("checkstyle:ParameterNumber")
private CompletableFuture<PushConnectionsHandle> connectToHosts(final ServiceContext serviceContext, final PushPhysicalPlanManager pushPhysicalPlanManager, final ConfiguredStatement<Query> statement, final Collection<KsqlNode> hosts, final LogicalSchema outputSchema, final TransientQueryQueue transientQueryQueue, final PushConnectionsHandle pushConnectionsHandle, final boolean dynamicallyAddedNode, final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics, final Set<KsqlNode> catchupHosts, final PushRoutingOptions pushRoutingOptions, final String thisHostName) {
    final Map<KsqlNode, CompletableFuture<RoutingResult>> futureMap = new LinkedHashMap<>();
    for (final KsqlNode node : hosts) {
        pushConnectionsHandle.add(node, new RoutingResult(RoutingResultStatus.IN_PROGRESS, () -> {
        }));
        final CompletableFuture<Void> callback = new CompletableFuture<>();
        callback.handle((v, t) -> {
            if (t == null) {
                pushConnectionsHandle.get(node).ifPresent(result -> {
                    result.close();
                    result.updateStatus(RoutingResultStatus.COMPLETE);
                });
                LOG.info("Host {} completed request {}.", node, pushPhysicalPlanManager.getQueryId());
            } else if (t instanceof GapFoundException) {
                pushConnectionsHandle.get(node).ifPresent(result -> {
                    result.close();
                    result.updateStatus(RoutingResultStatus.OFFSET_GAP_FOUND);
                });
            } else {
                pushConnectionsHandle.completeExceptionally(t);
            }
            return null;
        });
        futureMap.put(node, executeOrRouteQuery(node, statement, serviceContext, pushPhysicalPlanManager, outputSchema, transientQueryQueue, callback, scalablePushQueryMetrics, pushConnectionsHandle.getOffsetsTracker(), catchupHosts.contains(node), pushRoutingOptions, thisHostName));
    }
    return CompletableFuture.allOf(futureMap.values().toArray(new CompletableFuture[0])).thenApply(v -> {
        for (final KsqlNode node : hosts) {
            final CompletableFuture<RoutingResult> future = futureMap.get(node);
            final RoutingResult routingResult = future.join();
            pushConnectionsHandle.add(node, routingResult);
        }
        return pushConnectionsHandle;
    }).exceptionally(t -> {
        final KsqlNode node = futureMap.entrySet().stream().filter(e -> e.getValue().isCompletedExceptionally()).map(Entry::getKey).findFirst().orElse(null);
        for (KsqlNode n : hosts) {
            final CompletableFuture<RoutingResult> future = futureMap.get(n);
            // Take whatever completed exceptionally and mark it as failed
            if (future.isCompletedExceptionally()) {
                pushConnectionsHandle.get(n).ifPresent(result -> result.updateStatus(RoutingResultStatus.FAILED));
            } else {
                final RoutingResult routingResult = future.join();
                pushConnectionsHandle.add(node, routingResult);
            }
        }
        LOG.warn("Error routing query {} id {} to host {} at timestamp {} with exception {}", statement.getStatementText(), pushPhysicalPlanManager.getQueryId(), node, System.currentTimeMillis(), t.getCause());
        // retries in that case and don't fail the original request.
        if (!dynamicallyAddedNode) {
            pushConnectionsHandle.completeExceptionally(new KsqlException(String.format("Unable to execute push query \"%s\". %s", statement.getStatementText(), t.getCause().getMessage())));
        }
        return pushConnectionsHandle;
    }).exceptionally(t -> {
        LOG.error("Unexpected error handing exception", t);
        return pushConnectionsHandle;
    });
}
Also used : Query(io.confluent.ksql.parser.tree.Query) LoadingCache(com.google.common.cache.LoadingCache) KeyValue(io.confluent.ksql.util.KeyValue) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) RowMetadata(io.confluent.ksql.util.RowMetadata) DataRow(io.confluent.ksql.rest.entity.StreamedRow.DataRow) RestResponse(io.confluent.ksql.rest.client.RestResponse) Context(io.vertx.core.Context) KsqlErrorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessage) PushContinuationToken(io.confluent.ksql.rest.entity.PushContinuationToken) QueryRow(io.confluent.ksql.physical.common.QueryRow) Map(java.util.Map) QueryId(io.confluent.ksql.query.QueryId) KsqlRequestConfig(io.confluent.ksql.util.KsqlRequestConfig) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) BufferedPublisher(io.confluent.ksql.reactive.BufferedPublisher) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) CacheLoader(com.google.common.cache.CacheLoader) KsqlNode(io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode) BaseSubscriber(io.confluent.ksql.reactive.BaseSubscriber) List(java.util.List) VertxUtils(io.confluent.ksql.util.VertxUtils) Entry(java.util.Map.Entry) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) CacheBuilder(com.google.common.cache.CacheBuilder) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) KeyValueMetadata(io.confluent.ksql.util.KeyValueMetadata) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange) OffsetVector(io.confluent.ksql.util.OffsetVector) BiConsumer(java.util.function.BiConsumer) Logger(org.slf4j.Logger) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) GenericRow(io.confluent.ksql.GenericRow) Subscription(org.reactivestreams.Subscription) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) PushOffsetVector(io.confluent.ksql.util.PushOffsetVector) Collections(java.util.Collections) CompletableFuture(java.util.concurrent.CompletableFuture) KsqlNode(io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode) KsqlException(io.confluent.ksql.util.KsqlException) LinkedHashMap(java.util.LinkedHashMap)

Example 2 with ScalablePushQueryMetrics

use of io.confluent.ksql.internal.ScalablePushQueryMetrics in project ksql by confluentinc.

the class PushRouting method checkForNewHosts.

private void checkForNewHosts(final ServiceContext serviceContext, final PushPhysicalPlanManager pushPhysicalPlanManager, final ConfiguredStatement<Query> statement, final LogicalSchema outputSchema, final TransientQueryQueue transientQueryQueue, final PushConnectionsHandle pushConnectionsHandle, final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics, final PushRoutingOptions pushRoutingOptions, final String thisHostName) {
    VertxUtils.checkContext(pushPhysicalPlanManager.getContext());
    if (pushConnectionsHandle.isClosed()) {
        return;
    }
    final Set<KsqlNode> updatedHosts = registryToNodes.apply(pushPhysicalPlanManager.getScalablePushRegistry());
    final Set<KsqlNode> hosts = pushConnectionsHandle.getActiveHosts();
    final Set<KsqlNode> newHosts = Sets.difference(updatedHosts, hosts).stream().filter(node -> pushConnectionsHandle.get(node).map(routingResult -> routingResult.getStatus() != RoutingResultStatus.IN_PROGRESS).orElse(true)).collect(Collectors.toSet());
    final Set<KsqlNode> removedHosts = Sets.difference(hosts, updatedHosts);
    if (newHosts.size() > 0) {
        LOG.info("Dynamically adding new hosts {} for {}", newHosts, pushPhysicalPlanManager.getQueryId());
        final Set<KsqlNode> catchupHosts = newHosts.stream().filter(node -> pushConnectionsHandle.get(node).map(routingResult -> routingResult.getStatus() == RoutingResultStatus.OFFSET_GAP_FOUND).orElse(false)).collect(Collectors.toSet());
        connectToHosts(serviceContext, pushPhysicalPlanManager, statement, newHosts, outputSchema, transientQueryQueue, pushConnectionsHandle, true, scalablePushQueryMetrics, catchupHosts, pushRoutingOptions, thisHostName);
    }
    if (removedHosts.size() > 0) {
        LOG.info("Dynamically removing hosts {} for {}", removedHosts, pushPhysicalPlanManager.getQueryId());
        for (final KsqlNode node : removedHosts) {
            final RoutingResult result = pushConnectionsHandle.remove(node);
            result.close();
            result.updateStatus(RoutingResultStatus.REMOVED);
        }
    }
    pushPhysicalPlanManager.getContext().owner().setTimer(clusterCheckInterval, timerId -> checkForNewHosts(serviceContext, pushPhysicalPlanManager, statement, outputSchema, transientQueryQueue, pushConnectionsHandle, scalablePushQueryMetrics, pushRoutingOptions, thisHostName));
}
Also used : Query(io.confluent.ksql.parser.tree.Query) LoadingCache(com.google.common.cache.LoadingCache) KeyValue(io.confluent.ksql.util.KeyValue) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) RowMetadata(io.confluent.ksql.util.RowMetadata) DataRow(io.confluent.ksql.rest.entity.StreamedRow.DataRow) RestResponse(io.confluent.ksql.rest.client.RestResponse) Context(io.vertx.core.Context) KsqlErrorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessage) PushContinuationToken(io.confluent.ksql.rest.entity.PushContinuationToken) QueryRow(io.confluent.ksql.physical.common.QueryRow) Map(java.util.Map) QueryId(io.confluent.ksql.query.QueryId) KsqlRequestConfig(io.confluent.ksql.util.KsqlRequestConfig) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) BufferedPublisher(io.confluent.ksql.reactive.BufferedPublisher) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) CacheLoader(com.google.common.cache.CacheLoader) KsqlNode(io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode) BaseSubscriber(io.confluent.ksql.reactive.BaseSubscriber) List(java.util.List) VertxUtils(io.confluent.ksql.util.VertxUtils) Entry(java.util.Map.Entry) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) CacheBuilder(com.google.common.cache.CacheBuilder) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) KeyValueMetadata(io.confluent.ksql.util.KeyValueMetadata) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange) OffsetVector(io.confluent.ksql.util.OffsetVector) BiConsumer(java.util.function.BiConsumer) Logger(org.slf4j.Logger) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) GenericRow(io.confluent.ksql.GenericRow) Subscription(org.reactivestreams.Subscription) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) PushOffsetVector(io.confluent.ksql.util.PushOffsetVector) Collections(java.util.Collections) KsqlNode(io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode)

Example 3 with ScalablePushQueryMetrics

use of io.confluent.ksql.internal.ScalablePushQueryMetrics in project ksql by confluentinc.

the class ScalablePushQueryMetricsTest method setUp.

@Before
public void setUp() {
    when(ksqlEngine.getServiceId()).thenReturn(KSQL_SERVICE_ID);
    when(time.nanoseconds()).thenReturn(6000L);
    scalablePushQueryMetrics = new ScalablePushQueryMetrics(ksqlEngine.getServiceId(), CUSTOM_TAGS, time, new Metrics());
}
Also used : ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) Metrics(org.apache.kafka.common.metrics.Metrics) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) Before(org.junit.Before)

Example 4 with ScalablePushQueryMetrics

use of io.confluent.ksql.internal.ScalablePushQueryMetrics in project ksql by confluentinc.

the class EngineExecutor method executeScalablePushQuery.

ScalablePushQueryMetadata executeScalablePushQuery(final ImmutableAnalysis analysis, final ConfiguredStatement<Query> statement, final PushRouting pushRouting, final PushRoutingOptions pushRoutingOptions, final QueryPlannerOptions queryPlannerOptions, final Context context, final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics) {
    final SessionConfig sessionConfig = statement.getSessionConfig();
    // If we ever change how many hops a request can do, we'll need to update this for correct
    // metrics.
    final RoutingNodeType routingNodeType = pushRoutingOptions.getHasBeenForwarded() ? RoutingNodeType.REMOTE_NODE : RoutingNodeType.SOURCE_NODE;
    PushPhysicalPlan plan = null;
    try {
        final KsqlConfig ksqlConfig = sessionConfig.getConfig(false);
        final LogicalPlanNode logicalPlan = buildAndValidateLogicalPlan(statement, analysis, ksqlConfig, queryPlannerOptions, true);
        final PushPhysicalPlanCreator pushPhysicalPlanCreator = (offsetRange, catchupConsumerGroup) -> buildScalablePushPhysicalPlan(logicalPlan, analysis, context, offsetRange, catchupConsumerGroup);
        final Optional<PushOffsetRange> offsetRange = pushRoutingOptions.getContinuationToken().map(PushOffsetRange::deserialize);
        final Optional<String> catchupConsumerGroup = pushRoutingOptions.getCatchupConsumerGroup();
        final PushPhysicalPlanManager physicalPlanManager = new PushPhysicalPlanManager(pushPhysicalPlanCreator, catchupConsumerGroup, offsetRange);
        final PushPhysicalPlan physicalPlan = physicalPlanManager.getPhysicalPlan();
        plan = physicalPlan;
        final TransientQueryQueue transientQueryQueue = new TransientQueryQueue(analysis.getLimitClause());
        final PushQueryMetadata.ResultType resultType = physicalPlan.getScalablePushRegistry().isTable() ? physicalPlan.getScalablePushRegistry().isWindowed() ? ResultType.WINDOWED_TABLE : ResultType.TABLE : ResultType.STREAM;
        final PushQueryQueuePopulator populator = () -> pushRouting.handlePushQuery(serviceContext, physicalPlanManager, statement, pushRoutingOptions, physicalPlan.getOutputSchema(), transientQueryQueue, scalablePushQueryMetrics, offsetRange);
        final PushQueryPreparer preparer = () -> pushRouting.preparePushQuery(physicalPlanManager, statement, pushRoutingOptions);
        final ScalablePushQueryMetadata metadata = new ScalablePushQueryMetadata(physicalPlan.getOutputSchema(), physicalPlan.getQueryId(), transientQueryQueue, scalablePushQueryMetrics, resultType, populator, preparer, physicalPlan.getSourceType(), routingNodeType, physicalPlan::getRowsReadFromDataSource);
        return metadata;
    } catch (final Exception e) {
        if (plan == null) {
            scalablePushQueryMetrics.ifPresent(m -> m.recordErrorRateForNoResult(1));
        } else {
            final PushPhysicalPlan pushPhysicalPlan = plan;
            scalablePushQueryMetrics.ifPresent(metrics -> metrics.recordErrorRate(1, pushPhysicalPlan.getSourceType(), routingNodeType));
        }
        final String stmtLower = statement.getStatementText().toLowerCase(Locale.ROOT);
        final String messageLower = e.getMessage().toLowerCase(Locale.ROOT);
        final String stackLower = Throwables.getStackTraceAsString(e).toLowerCase(Locale.ROOT);
        // the contents of the query
        if (messageLower.contains(stmtLower) || stackLower.contains(stmtLower)) {
            final StackTraceElement loc = Iterables.getLast(Throwables.getCausalChain(e)).getStackTrace()[0];
            LOG.error("Failure to execute push query V2 {} {}, not logging the error message since it " + "contains the query string, which may contain sensitive information." + " If you see this LOG message, please submit a GitHub ticket and" + " we will scrub the statement text from the error at {}", pushRoutingOptions.debugString(), queryPlannerOptions.debugString(), loc);
        } else {
            LOG.error("Failure to execute push query V2. {} {}", pushRoutingOptions.debugString(), queryPlannerOptions.debugString(), e);
        }
        LOG.debug("Failed push query V2 text {}, {}", statement.getStatementText(), e);
        throw new KsqlStatementException(e.getMessage() == null ? "Server Error" + Arrays.toString(e.getStackTrace()) : e.getMessage(), statement.getStatementText(), e);
    }
}
Also used : DataSource(io.confluent.ksql.metastore.model.DataSource) PushPhysicalPlanCreator(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanCreator) CreateTableAsSelect(io.confluent.ksql.parser.tree.CreateTableAsSelect) Arrays(java.util.Arrays) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) SourceName(io.confluent.ksql.name.SourceName) RoutingOptions(io.confluent.ksql.execution.streams.RoutingOptions) PushPhysicalPlanManager(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanManager) PushPhysicalPlanBuilder(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanBuilder) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) TransientQueryMetadata(io.confluent.ksql.util.TransientQueryMetadata) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) Map(java.util.Map) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) QueryId(io.confluent.ksql.query.QueryId) ExecutionStep(io.confluent.ksql.execution.plan.ExecutionStep) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) Sink(io.confluent.ksql.parser.tree.Sink) Set(java.util.Set) Relation(io.confluent.ksql.parser.tree.Relation) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) KsqlException(io.confluent.ksql.util.KsqlException) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) PullQueryResult(io.confluent.ksql.physical.pull.PullQueryResult) Iterables(com.google.common.collect.Iterables) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) PushRouting(io.confluent.ksql.physical.scalablepush.PushRouting) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) SessionConfig(io.confluent.ksql.config.SessionConfig) CreateStream(io.confluent.ksql.parser.tree.CreateStream) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) MetaStore(io.confluent.ksql.metastore.MetaStore) KsqlStructuredDataOutputNode(io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode) PushRoutingOptions(io.confluent.ksql.physical.scalablepush.PushRoutingOptions) PlanInfoExtractor(io.confluent.ksql.execution.plan.PlanInfoExtractor) DataSourceNode(io.confluent.ksql.planner.plan.DataSourceNode) QueryContainer(io.confluent.ksql.parser.tree.QueryContainer) OutputNode(io.confluent.ksql.planner.plan.OutputNode) Throwables(com.google.common.base.Throwables) PushQueryMetadata(io.confluent.ksql.util.PushQueryMetadata) PushQueryQueuePopulator(io.confluent.ksql.physical.scalablepush.PushQueryQueuePopulator) ValueFormat(io.confluent.ksql.serde.ValueFormat) Table(io.confluent.ksql.parser.tree.Table) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) OutputRefinement(io.confluent.ksql.parser.OutputRefinement) LogicalPlanNode(io.confluent.ksql.planner.LogicalPlanNode) Query(io.confluent.ksql.parser.tree.Query) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) Formats(io.confluent.ksql.execution.plan.Formats) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) Context(io.vertx.core.Context) CreateTable(io.confluent.ksql.parser.tree.CreateTable) Locale(java.util.Locale) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) TopicPartition(org.apache.kafka.common.TopicPartition) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) KsqlConfig(io.confluent.ksql.util.KsqlConfig) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) DdlCommand(io.confluent.ksql.execution.ddl.commands.DdlCommand) Objects(java.util.Objects) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) QueryPlannerOptions(io.confluent.ksql.planner.QueryPlannerOptions) ConsistencyOffsetVector(io.confluent.ksql.util.ConsistencyOffsetVector) Optional(java.util.Optional) Statement(io.confluent.ksql.parser.tree.Statement) KsqlConstants(io.confluent.ksql.util.KsqlConstants) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) PullQueryQueuePopulator(io.confluent.ksql.physical.pull.PullQueryQueuePopulator) PullQueryQueue(io.confluent.ksql.query.PullQueryQueue) PushPhysicalPlan(io.confluent.ksql.physical.scalablepush.PushPhysicalPlan) HARouting(io.confluent.ksql.physical.pull.HARouting) PlanInfo(io.confluent.ksql.execution.plan.PlanInfo) PullPhysicalPlanBuilder(io.confluent.ksql.physical.pull.PullPhysicalPlanBuilder) KeyFormat(io.confluent.ksql.serde.KeyFormat) ResultType(io.confluent.ksql.util.PushQueryMetadata.ResultType) CompletableFuture(java.util.concurrent.CompletableFuture) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) OptionalInt(java.util.OptionalInt) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange) LogicalPlanner(io.confluent.ksql.planner.LogicalPlanner) Logger(org.slf4j.Logger) PhysicalPlan(io.confluent.ksql.physical.PhysicalPlan) PlanSummary(io.confluent.ksql.util.PlanSummary) PullPhysicalPlan(io.confluent.ksql.physical.pull.PullPhysicalPlan) PlanNode(io.confluent.ksql.planner.plan.PlanNode) QueryRegistry(io.confluent.ksql.query.QueryRegistry) Collections(java.util.Collections) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Select(io.confluent.ksql.parser.tree.Select) PushQueryPreparer(io.confluent.ksql.physical.scalablepush.PushQueryPreparer) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) PushQueryMetadata(io.confluent.ksql.util.PushQueryMetadata) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) PushPhysicalPlan(io.confluent.ksql.physical.scalablepush.PushPhysicalPlan) SessionConfig(io.confluent.ksql.config.SessionConfig) KsqlConfig(io.confluent.ksql.util.KsqlConfig) LogicalPlanNode(io.confluent.ksql.planner.LogicalPlanNode) KsqlException(io.confluent.ksql.util.KsqlException) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) PushQueryPreparer(io.confluent.ksql.physical.scalablepush.PushQueryPreparer) PushPhysicalPlanManager(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanManager) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) PushPhysicalPlanCreator(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanCreator) PushQueryQueuePopulator(io.confluent.ksql.physical.scalablepush.PushQueryQueuePopulator) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) ResultType(io.confluent.ksql.util.PushQueryMetadata.ResultType) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange)

Example 5 with ScalablePushQueryMetrics

use of io.confluent.ksql.internal.ScalablePushQueryMetrics in project ksql by confluentinc.

the class QueryMetricsUtil method initializeScalablePushMetricsCallback.

public static MetricsCallback initializeScalablePushMetricsCallback(final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics, final SlidingWindowRateLimiter scalablePushBandRateLimiter, final AtomicReference<ScalablePushQueryMetadata> resultForMetrics) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> scalablePushQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final ScalablePushQueryMetadata r = resultForMetrics.get();
        if (r == null) {
            metrics.recordResponseSizeForError(responseBytes);
            metrics.recordConnectionDurationForError(startTimeNanos);
            metrics.recordZeroRowsReturnedForError();
            metrics.recordZeroRowsProcessedForError();
        } else {
            final QuerySourceType sourceType = r.getSourceType();
            final RoutingNodeType routingNodeType = r.getRoutingNodeType();
            metrics.recordResponseSize(responseBytes, sourceType, routingNodeType);
            metrics.recordConnectionDuration(startTimeNanos, sourceType, routingNodeType);
            metrics.recordRowsReturned(r.getTotalRowsReturned(), sourceType, routingNodeType);
            metrics.recordRowsProcessed(r.getTotalRowsProcessed(), sourceType, routingNodeType);
        }
        scalablePushBandRateLimiter.add(responseBytes);
    });
    return metricsCallback;
}
Also used : ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) Decrementer(io.confluent.ksql.rest.util.ConcurrencyLimiter.Decrementer) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) QuerySourceType(io.confluent.ksql.util.KsqlConstants.QuerySourceType) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) MetricsCallback(io.confluent.ksql.api.server.MetricsCallback) State(org.apache.kafka.streams.KafkaStreams.State) AtomicReference(java.util.concurrent.atomic.AtomicReference) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) StreamPullQueryMetadata(io.confluent.ksql.util.StreamPullQueryMetadata) Optional(java.util.Optional) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) KafkaStreams(org.apache.kafka.streams.KafkaStreams) PullPhysicalPlanType(io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType) PullQueryResult(io.confluent.ksql.physical.pull.PullQueryResult) SlidingWindowRateLimiter(io.confluent.ksql.api.server.SlidingWindowRateLimiter) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) MetricsCallback(io.confluent.ksql.api.server.MetricsCallback) QuerySourceType(io.confluent.ksql.util.KsqlConstants.QuerySourceType)

Aggregations

ScalablePushQueryMetrics (io.confluent.ksql.internal.ScalablePushQueryMetrics)7 PullQueryExecutorMetrics (io.confluent.ksql.internal.PullQueryExecutorMetrics)4 Query (io.confluent.ksql.parser.tree.Query)4 QueryId (io.confluent.ksql.query.QueryId)4 TransientQueryQueue (io.confluent.ksql.query.TransientQueryQueue)4 Optional (java.util.Optional)4 ImmutableMap (com.google.common.collect.ImmutableMap)3 ImmutableSet (com.google.common.collect.ImmutableSet)3 SuppressFBWarnings (edu.umd.cs.findbugs.annotations.SuppressFBWarnings)3 LogicalSchema (io.confluent.ksql.schema.ksql.LogicalSchema)3 ServiceContext (io.confluent.ksql.services.ServiceContext)3 ConfiguredStatement (io.confluent.ksql.statement.ConfiguredStatement)3 KsqlException (io.confluent.ksql.util.KsqlException)3 PushOffsetRange (io.confluent.ksql.util.PushOffsetRange)3 Context (io.vertx.core.Context)3 Collection (java.util.Collection)3 Collections (java.util.Collections)3 Map (java.util.Map)3 Set (java.util.Set)3 CompletableFuture (java.util.concurrent.CompletableFuture)3