Examples with HARouting io.confluent.ksql.physical.pull.HARouting used on opensource projects

Example 1 with HARouting

use of io.confluent.ksql.physical.pull.HARouting in project ksql by confluentinc.

the class EngineExecutor method executeTablePullQuery.

/**
 * Evaluates a pull query by first analyzing it, then building the logical plan and finally
 * the physical plan. The execution is then done using the physical plan in a pipelined manner.
 * @param statement The pull query
 * @param routingOptions Configuration parameters used for HA routing
 * @param pullQueryMetrics JMX metrics
 * @return the rows that are the result of evaluating the pull query
 */
PullQueryResult executeTablePullQuery(final ImmutableAnalysis analysis, final ConfiguredStatement<Query> statement, final HARouting routing, final RoutingOptions routingOptions, final QueryPlannerOptions queryPlannerOptions, final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final boolean startImmediately, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    if (!statement.getStatement().isPullQuery()) {
        throw new IllegalArgumentException("Executor can only handle pull queries");
    }
    final SessionConfig sessionConfig = statement.getSessionConfig();
    // If we ever change how many hops a request can do, we'll need to update this for correct
    // metrics.
    final RoutingNodeType routingNodeType = routingOptions.getIsSkipForwardRequest() ? RoutingNodeType.REMOTE_NODE : RoutingNodeType.SOURCE_NODE;
    PullPhysicalPlan plan = null;
    try {
        // Do not set sessionConfig.getConfig to true! The copying is inefficient and slows down pull
        // query performance significantly.  Instead use QueryPlannerOptions which check overrides
        // deliberately.
        final KsqlConfig ksqlConfig = sessionConfig.getConfig(false);
        final LogicalPlanNode logicalPlan = buildAndValidateLogicalPlan(statement, analysis, ksqlConfig, queryPlannerOptions, false);
        // This is a cancel signal that is used to stop both local operations and requests
        final CompletableFuture<Void> shouldCancelRequests = new CompletableFuture<>();
        plan = buildPullPhysicalPlan(logicalPlan, analysis, queryPlannerOptions, shouldCancelRequests, consistencyOffsetVector);
        final PullPhysicalPlan physicalPlan = plan;
        final PullQueryQueue pullQueryQueue = new PullQueryQueue(analysis.getLimitClause());
        final PullQueryQueuePopulator populator = () -> routing.handlePullQuery(serviceContext, physicalPlan, statement, routingOptions, physicalPlan.getOutputSchema(), physicalPlan.getQueryId(), pullQueryQueue, shouldCancelRequests, consistencyOffsetVector);
        final PullQueryResult result = new PullQueryResult(physicalPlan.getOutputSchema(), populator, physicalPlan.getQueryId(), pullQueryQueue, pullQueryMetrics, physicalPlan.getSourceType(), physicalPlan.getPlanType(), routingNodeType, physicalPlan::getRowsReadFromDataSource, shouldCancelRequests, consistencyOffsetVector);
        if (startImmediately) {
            result.start();
        }
        return result;
    } catch (final Exception e) {
        if (plan == null) {
            pullQueryMetrics.ifPresent(m -> m.recordErrorRateForNoResult(1));
        } else {
            final PullPhysicalPlan physicalPlan = plan;
            pullQueryMetrics.ifPresent(metrics -> metrics.recordErrorRate(1, physicalPlan.getSourceType(), physicalPlan.getPlanType(), routingNodeType));
        }
        final String stmtLower = statement.getStatementText().toLowerCase(Locale.ROOT);
        final String messageLower = e.getMessage().toLowerCase(Locale.ROOT);
        final String stackLower = Throwables.getStackTraceAsString(e).toLowerCase(Locale.ROOT);
        // the contents of the query
        if (messageLower.contains(stmtLower) || stackLower.contains(stmtLower)) {
            final StackTraceElement loc = Iterables.getLast(Throwables.getCausalChain(e)).getStackTrace()[0];
            LOG.error("Failure to execute pull query {} {}, not logging the error message since it " + "contains the query string, which may contain sensitive information. If you " + "see this LOG message, please submit a GitHub ticket and we will scrub " + "the statement text from the error at {}", routingOptions.debugString(), queryPlannerOptions.debugString(), loc);
        } else {
            LOG.error("Failure to execute pull query. {} {}", routingOptions.debugString(), queryPlannerOptions.debugString(), e);
        }
        LOG.debug("Failed pull query text {}, {}", statement.getStatementText(), e);
        throw new KsqlStatementException(e.getMessage() == null ? "Server Error" + Arrays.toString(e.getStackTrace()) : e.getMessage(), statement.getStatementText(), e);
    }
}

Example 2 with HARouting

use of io.confluent.ksql.physical.pull.HARouting in project ksql by confluentinc.

the class KsqlRestApplication method buildApplication.

@SuppressWarnings({ "checkstyle:JavaNCSS", "checkstyle:MethodLength", "checkstyle:ParameterNumber" })
static KsqlRestApplication buildApplication(final String metricsPrefix, final KsqlRestConfig restConfig, final Function<Supplier<Boolean>, VersionCheckerAgent> versionCheckerFactory, final int maxStatementRetries, final ServiceContext serviceContext, final Supplier<SchemaRegistryClient> schemaRegistryClientFactory, final ConnectClientFactory connectClientFactory, final Vertx vertx, final KsqlClient sharedClient, final DefaultServiceContextFactory defaultServiceContextFactory, final UserServiceContextFactory userServiceContextFactory, final MetricCollectors metricCollectors) {
    final String ksqlInstallDir = restConfig.getString(KsqlRestConfig.INSTALL_DIR_CONFIG);
    final KsqlConfig ksqlConfig = new KsqlConfig(restConfig.getKsqlConfigProperties());
    final ProcessingLogConfig processingLogConfig = new ProcessingLogConfig(restConfig.getOriginals());
    final ProcessingLogContext processingLogContext = ProcessingLogContext.create(processingLogConfig);
    final MutableFunctionRegistry functionRegistry = new InternalFunctionRegistry();
    if (restConfig.getBoolean(KsqlRestConfig.KSQL_SERVER_ENABLE_UNCAUGHT_EXCEPTION_HANDLER)) {
        Thread.setDefaultUncaughtExceptionHandler(new KsqlUncaughtExceptionHandler(LogManager::shutdown));
    }
    final SpecificQueryIdGenerator specificQueryIdGenerator = new SpecificQueryIdGenerator();
    final String stateDir = ksqlConfig.getKsqlStreamConfigProps().getOrDefault(StreamsConfig.STATE_DIR_CONFIG, StreamsConfig.configDef().defaultValues().get(StreamsConfig.STATE_DIR_CONFIG)).toString();
    final ServiceInfo serviceInfo = ServiceInfo.create(ksqlConfig, metricsPrefix);
    final Map<String, String> metricsTags = ImmutableMap.<String, String>builder().putAll(serviceInfo.customMetricsTags()).put(KsqlConstants.KSQL_SERVICE_ID_METRICS_TAG, serviceInfo.serviceId()).build();
    StorageUtilizationMetricsReporter.configureShared(new File(stateDir), metricCollectors.getMetrics(), ksqlConfig.getStringAsMap(KsqlConfig.KSQL_CUSTOM_METRICS_TAGS));
    final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("ksql-csu-metrics-reporter-%d").build());
    final ScheduledExecutorService leakedResourcesReporter = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("ksql-leaked-resources-metrics-reporter-%d").build());
    final KsqlEngine ksqlEngine = new KsqlEngine(serviceContext, processingLogContext, functionRegistry, serviceInfo, specificQueryIdGenerator, new KsqlConfig(restConfig.getKsqlConfigProperties()), Collections.emptyList(), metricCollectors);
    final PersistentQuerySaturationMetrics saturation = new PersistentQuerySaturationMetrics(ksqlEngine, new JmxDataPointsReporter(metricCollectors.getMetrics(), "ksqldb_utilization", Duration.ofMinutes(1)), Duration.ofMinutes(5), Duration.ofSeconds(30), ksqlConfig.getStringAsMap(KsqlConfig.KSQL_CUSTOM_METRICS_TAGS));
    executorService.scheduleAtFixedRate(saturation, 0, Duration.ofMinutes(1).toMillis(), TimeUnit.MILLISECONDS);
    final int transientQueryCleanupServicePeriod = ksqlConfig.getInt(KsqlConfig.KSQL_TRANSIENT_QUERY_CLEANUP_SERVICE_PERIOD_SECONDS);
    final LeakedResourcesMetrics leaked = new LeakedResourcesMetrics(ksqlEngine, new JmxDataPointsReporter(metricCollectors.getMetrics(), ReservedInternalTopics.KSQL_INTERNAL_TOPIC_PREFIX + ksqlConfig.getString(KsqlConfig.KSQL_SERVICE_ID_CONFIG) + ".leaked_resources_metrics", Duration.ofSeconds(transientQueryCleanupServicePeriod)), ksqlConfig.getStringAsMap(KsqlConfig.KSQL_CUSTOM_METRICS_TAGS));
    leakedResourcesReporter.scheduleAtFixedRate(leaked, 0, transientQueryCleanupServicePeriod, TimeUnit.SECONDS);
    UserFunctionLoader.newInstance(ksqlConfig, functionRegistry, ksqlInstallDir, metricCollectors.getMetrics()).load();
    final String commandTopicName = ReservedInternalTopics.commandTopic(ksqlConfig);
    final Admin internalAdmin = createCommandTopicAdminClient(restConfig, ksqlConfig);
    final KafkaTopicClient internalTopicClient = new KafkaTopicClientImpl(() -> internalAdmin);
    final CommandStore commandStore = CommandStore.Factory.create(ksqlConfig, commandTopicName, Duration.ofMillis(restConfig.getLong(DISTRIBUTED_COMMAND_RESPONSE_TIMEOUT_MS_CONFIG)), ksqlConfig.addConfluentMetricsContextConfigsKafka(restConfig.getCommandConsumerProperties()), ksqlConfig.addConfluentMetricsContextConfigsKafka(restConfig.getCommandProducerProperties()), internalTopicClient);
    final InteractiveStatementExecutor statementExecutor = new InteractiveStatementExecutor(serviceContext, ksqlEngine, specificQueryIdGenerator);
    final StatusResource statusResource = new StatusResource(statementExecutor);
    final VersionCheckerAgent versionChecker = versionCheckerFactory.apply(ksqlEngine::hasActiveQueries);
    final ServerState serverState = new ServerState();
    final KsqlSecurityExtension securityExtension = loadSecurityExtension(ksqlConfig);
    final KsqlSecurityContextProvider ksqlSecurityContextProvider = new DefaultKsqlSecurityContextProvider(securityExtension, defaultServiceContextFactory, userServiceContextFactory, ksqlConfig, schemaRegistryClientFactory, connectClientFactory, sharedClient);
    final Optional<AuthenticationPlugin> securityHandlerPlugin = loadAuthenticationPlugin(restConfig);
    final Optional<KsqlAuthorizationValidator> authorizationValidator = KsqlAuthorizationValidatorFactory.create(ksqlConfig, serviceContext, securityExtension.getAuthorizationProvider());
    final Errors errorHandler = new Errors(restConfig.getConfiguredInstance(KsqlRestConfig.KSQL_SERVER_ERROR_MESSAGES, ErrorMessages.class));
    final ConnectServerErrors connectErrorHandler = loadConnectErrorHandler(ksqlConfig);
    final Optional<LagReportingAgent> lagReportingAgent = initializeLagReportingAgent(restConfig, ksqlEngine, serviceContext);
    final Optional<HeartbeatAgent> heartbeatAgent = initializeHeartbeatAgent(restConfig, ksqlEngine, serviceContext, lagReportingAgent);
    final RoutingFilterFactory routingFilterFactory = initializeRoutingFilterFactory(ksqlConfig, heartbeatAgent, lagReportingAgent);
    final RateLimiter pullQueryRateLimiter = new RateLimiter(ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PULL_MAX_QPS_CONFIG), "pull", metricCollectors.getMetrics(), metricsTags);
    final ConcurrencyLimiter pullQueryConcurrencyLimiter = new ConcurrencyLimiter(ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PULL_MAX_CONCURRENT_REQUESTS_CONFIG), "pull", metricCollectors.getMetrics(), metricsTags);
    final SlidingWindowRateLimiter pullBandRateLimiter = new SlidingWindowRateLimiter(ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PULL_MAX_HOURLY_BANDWIDTH_MEGABYTES_CONFIG), NUM_MILLISECONDS_IN_HOUR, "pull", metricCollectors.getMetrics(), metricsTags);
    final SlidingWindowRateLimiter scalablePushBandRateLimiter = new SlidingWindowRateLimiter(ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PUSH_V2_MAX_HOURLY_BANDWIDTH_MEGABYTES_CONFIG), NUM_MILLISECONDS_IN_HOUR, "push", metricCollectors.getMetrics(), metricsTags);
    final DenyListPropertyValidator denyListPropertyValidator = new DenyListPropertyValidator(ksqlConfig.getList(KsqlConfig.KSQL_PROPERTIES_OVERRIDES_DENYLIST));
    final Optional<PullQueryExecutorMetrics> pullQueryMetrics = ksqlConfig.getBoolean(KsqlConfig.KSQL_QUERY_PULL_METRICS_ENABLED) ? Optional.of(new PullQueryExecutorMetrics(ksqlEngine.getServiceId(), ksqlConfig.getStringAsMap(KsqlConfig.KSQL_CUSTOM_METRICS_TAGS), Time.SYSTEM, metricCollectors.getMetrics())) : Optional.empty();
    final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics = ksqlConfig.getBoolean(KsqlConfig.KSQL_QUERY_PUSH_V2_ENABLED) ? Optional.of(new ScalablePushQueryMetrics(ksqlEngine.getServiceId(), ksqlConfig.getStringAsMap(KsqlConfig.KSQL_CUSTOM_METRICS_TAGS), Time.SYSTEM, metricCollectors.getMetrics())) : Optional.empty();
    final HARouting pullQueryRouting = new HARouting(routingFilterFactory, pullQueryMetrics, ksqlConfig);
    final PushRouting pushQueryRouting = new PushRouting();
    final Optional<LocalCommands> localCommands = createLocalCommands(restConfig, ksqlEngine);
    final QueryExecutor queryExecutor = new QueryExecutor(ksqlEngine, restConfig, ksqlConfig, pullQueryMetrics, scalablePushQueryMetrics, pullQueryRateLimiter, pullQueryConcurrencyLimiter, pullBandRateLimiter, scalablePushBandRateLimiter, pullQueryRouting, pushQueryRouting, localCommands);
    final StreamedQueryResource streamedQueryResource = new StreamedQueryResource(ksqlEngine, restConfig, commandStore, Duration.ofMillis(restConfig.getLong(KsqlRestConfig.STREAMED_QUERY_DISCONNECT_CHECK_MS_CONFIG)), Duration.ofMillis(restConfig.getLong(DISTRIBUTED_COMMAND_RESPONSE_TIMEOUT_MS_CONFIG)), versionChecker::updateLastRequestTime, authorizationValidator, errorHandler, denyListPropertyValidator, queryExecutor);
    final List<String> managedTopics = new LinkedList<>();
    managedTopics.add(commandTopicName);
    if (processingLogConfig.getBoolean(ProcessingLogConfig.TOPIC_AUTO_CREATE)) {
        managedTopics.add(ProcessingLogServerUtils.getTopicName(processingLogConfig, ksqlConfig));
    }
    final CommandRunner commandRunner = new CommandRunner(statementExecutor, commandStore, maxStatementRetries, new ClusterTerminator(ksqlEngine, serviceContext, managedTopics), serverState, ksqlConfig.getString(KsqlConfig.KSQL_SERVICE_ID_CONFIG), Duration.ofMillis(restConfig.getLong(KsqlRestConfig.KSQL_COMMAND_RUNNER_BLOCKED_THRESHHOLD_ERROR_MS)), metricsPrefix, InternalTopicSerdes.deserializer(Command.class), errorHandler, internalTopicClient, commandTopicName, metricCollectors.getMetrics());
    final KsqlResource ksqlResource = new KsqlResource(ksqlEngine, commandRunner, Duration.ofMillis(restConfig.getLong(DISTRIBUTED_COMMAND_RESPONSE_TIMEOUT_MS_CONFIG)), versionChecker::updateLastRequestTime, authorizationValidator, errorHandler, connectErrorHandler, denyListPropertyValidator);
    final List<KsqlServerPrecondition> preconditions = restConfig.getConfiguredInstances(KsqlRestConfig.KSQL_SERVER_PRECONDITIONS, KsqlServerPrecondition.class);
    final List<KsqlConfigurable> configurables = ImmutableList.of(ksqlResource, streamedQueryResource, statementExecutor);
    final Consumer<KsqlConfig> rocksDBConfigSetterHandler = RocksDBConfigSetterHandler::maybeConfigureRocksDBConfigSetter;
    return new KsqlRestApplication(serviceContext, ksqlEngine, ksqlConfig, restConfig, commandRunner, commandStore, statusResource, streamedQueryResource, ksqlResource, versionChecker, ksqlSecurityContextProvider, securityExtension, securityHandlerPlugin, serverState, processingLogContext, preconditions, configurables, rocksDBConfigSetterHandler, heartbeatAgent, lagReportingAgent, vertx, denyListPropertyValidator, pullQueryMetrics, scalablePushQueryMetrics, localCommands, queryExecutor, metricCollectors, internalTopicClient, internalAdmin);
}