Examples with QueryType com.facebook.presto.spi.resourceGroups.QueryType used on opensource projects

Example 1 with QueryType

use of com.facebook.presto.spi.resourceGroups.QueryType in project presto by prestodb.

the class PrestoSparkQueryPlanner method createQueryPlan.

public PlanAndMore createQueryPlan(Session session, PreparedQuery preparedQuery, WarningCollector warningCollector) {
    PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
    Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.of(queryExplainer), preparedQuery.getParameters(), warningCollector);
    LogicalPlanner logicalPlanner = new LogicalPlanner(false, session, optimizers.getPlanningTimeOptimizers(), idAllocator, metadata, sqlParser, statsCalculator, costCalculator, warningCollector, planChecker);
    Analysis analysis = analyzer.analyze(preparedQuery.getStatement());
    Plan plan = logicalPlanner.plan(analysis, OPTIMIZED_AND_VALIDATED);
    List<Input> inputs = new InputExtractor(metadata, session).extractInputs(plan.getRoot());
    Optional<Output> output = new OutputExtractor().extractOutput(plan.getRoot());
    Optional<QueryType> queryType = getQueryType(preparedQuery.getStatement().getClass());
    List<String> columnNames = ((OutputNode) plan.getRoot()).getColumnNames();
    return new PlanAndMore(plan, Optional.ofNullable(analysis.getUpdateType()), columnNames, ImmutableSet.copyOf(inputs), output, queryType);
}

Example 2 with QueryType

use of com.facebook.presto.spi.resourceGroups.QueryType in project presto by prestodb.

the class QueryStateMachine method getQueryInfo.

public QueryInfo getQueryInfo(Optional<StageInfo> rootStage) {
    // Query state must be captured first in order to provide a
    // correct view of the query.  For example, building this
    // information, the query could finish, and the task states would
    // never be visible.
    QueryState state = queryState.get();
    ExecutionFailureInfo failureCause = null;
    ErrorCode errorCode = null;
    if (state == QueryState.FAILED) {
        failureCause = this.failureCause.get();
        if (failureCause != null) {
            errorCode = failureCause.getErrorCode();
        }
    }
    boolean completeInfo = getAllStages(rootStage).stream().allMatch(StageInfo::isFinalStageInfo);
    Optional<List<TaskId>> failedTasks;
    // Traversing all tasks is expensive, thus only construct failedTasks list when query finished.
    if (state.isDone()) {
        failedTasks = Optional.of(getAllStages(rootStage).stream().flatMap(stageInfo -> Streams.concat(ImmutableList.of(stageInfo.getLatestAttemptExecutionInfo()).stream(), stageInfo.getPreviousAttemptsExecutionInfos().stream())).flatMap(execution -> execution.getTasks().stream()).filter(taskInfo -> taskInfo.getTaskStatus().getState() == TaskState.FAILED).map(TaskInfo::getTaskId).collect(toImmutableList()));
    } else {
        failedTasks = Optional.empty();
    }
    List<StageId> runtimeOptimizedStages = getAllStages(rootStage).stream().filter(StageInfo::isRuntimeOptimized).map(StageInfo::getStageId).collect(toImmutableList());
    QueryStats queryStats = getQueryStats(rootStage);
    return new QueryInfo(queryId, session.toSessionRepresentation(), state, memoryPool.get().getId(), queryStats.isScheduled(), self, outputManager.getQueryOutputInfo().map(QueryOutputInfo::getColumnNames).orElse(ImmutableList.of()), query, expandedQuery.get(), preparedQuery, queryStats, Optional.ofNullable(setCatalog.get()), Optional.ofNullable(setSchema.get()), setSessionProperties, resetSessionProperties, setRoles, addedPreparedStatements, deallocatedPreparedStatements, Optional.ofNullable(startedTransactionId.get()), clearTransactionId.get(), updateType.get(), rootStage, failureCause, errorCode, warningCollector.getWarnings(), inputs.get(), output.get(), completeInfo, Optional.of(resourceGroup), queryType, failedTasks, runtimeOptimizedStages.isEmpty() ? Optional.empty() : Optional.of(runtimeOptimizedStages), addedSessionFunctions, removedSessionFunctions);
}

Example 3 with QueryType

use of com.facebook.presto.spi.resourceGroups.QueryType in project presto by prestodb.

the class QueryPreparer method prepareQuery.

public PreparedQuery prepareQuery(Session session, Statement wrappedStatement, WarningCollector warningCollector) throws ParsingException, PrestoException, SemanticException {
    Statement statement = wrappedStatement;
    Optional<String> prepareSql = Optional.empty();
    if (statement instanceof Execute) {
        prepareSql = Optional.of(session.getPreparedStatementFromExecute((Execute) statement));
        statement = sqlParser.createStatement(prepareSql.get(), createParsingOptions(session, warningCollector));
    }
    if (statement instanceof Explain && ((Explain) statement).isAnalyze()) {
        Statement innerStatement = ((Explain) statement).getStatement();
        Optional<QueryType> innerQueryType = StatementUtils.getQueryType(innerStatement.getClass());
        if (!innerQueryType.isPresent() || innerQueryType.get() == QueryType.DATA_DEFINITION) {
            throw new PrestoException(NOT_SUPPORTED, "EXPLAIN ANALYZE doesn't support statement type: " + innerStatement.getClass().getSimpleName());
        }
    }
    List<Expression> parameters = ImmutableList.of();
    if (wrappedStatement instanceof Execute) {
        parameters = ((Execute) wrappedStatement).getParameters();
    }
    validateParameters(statement, parameters);
    Optional<String> formattedQuery = Optional.empty();
    if (isLogFormattedQueryEnabled(session)) {
        formattedQuery = Optional.of(getFormattedQuery(statement, parameters));
    }
    return new PreparedQuery(statement, parameters, formattedQuery, prepareSql);
}

Example 4 with QueryType

use of com.facebook.presto.spi.resourceGroups.QueryType in project presto by prestodb.

the class DispatchManager method createQueryInternal.

/**
 *  Creates and registers a dispatch query with the query tracker.  This method will never fail to register a query with the query
 *  tracker.  If an error occurs while, creating a dispatch query a failed dispatch will be created and registered.
 */
private <C> void createQueryInternal(QueryId queryId, String slug, int retryCount, SessionContext sessionContext, String query, ResourceGroupManager<C> resourceGroupManager) {
    Session session = null;
    PreparedQuery preparedQuery;
    try {
        if (query.length() > maxQueryLength) {
            int queryLength = query.length();
            query = query.substring(0, maxQueryLength);
            throw new PrestoException(QUERY_TEXT_TOO_LARGE, format("Query text length (%s) exceeds the maximum length (%s)", queryLength, maxQueryLength));
        }
        // decode session
        session = sessionSupplier.createSession(queryId, sessionContext);
        // prepare query
        WarningCollector warningCollector = warningCollectorFactory.create(getWarningHandlingLevel(session));
        preparedQuery = queryPreparer.prepareQuery(session, query, warningCollector);
        query = preparedQuery.getFormattedQuery().orElse(query);
        // select resource group
        Optional<QueryType> queryType = getQueryType(preparedQuery.getStatement().getClass());
        SelectionContext<C> selectionContext = resourceGroupManager.selectGroup(new SelectionCriteria(sessionContext.getIdentity().getPrincipal().isPresent(), sessionContext.getIdentity().getUser(), Optional.ofNullable(sessionContext.getSource()), sessionContext.getClientTags(), sessionContext.getResourceEstimates(), queryType.map(Enum::name)));
        // apply system default session properties (does not override user set properties)
        session = sessionPropertyDefaults.newSessionWithDefaultProperties(session, queryType.map(Enum::name), Optional.of(selectionContext.getResourceGroupId()));
        // mark existing transaction as active
        transactionManager.activateTransaction(session, isTransactionControlStatement(preparedQuery.getStatement()), accessControl);
        DispatchQuery dispatchQuery = dispatchQueryFactory.createDispatchQuery(session, query, preparedQuery, slug, retryCount, selectionContext.getResourceGroupId(), queryType, warningCollector, (dq) -> resourceGroupManager.submit(preparedQuery.getStatement(), dq, selectionContext, queryExecutor));
        boolean queryAdded = queryCreated(dispatchQuery);
        if (queryAdded && !dispatchQuery.isDone()) {
            try {
                clusterStatusSender.registerQuery(dispatchQuery);
                dispatchQuery.startWaitingForPrerequisites();
            } catch (Throwable e) {
                // dispatch query has already been registered, so just fail it directly
                dispatchQuery.fail(e);
            }
        }
    } catch (Throwable throwable) {
        // creation must never fail, so register a failed query in this case
        if (session == null) {
            session = Session.builder(new SessionPropertyManager()).setQueryId(queryId).setIdentity(sessionContext.getIdentity()).setSource(sessionContext.getSource()).build();
        }
        DispatchQuery failedDispatchQuery = failedDispatchQueryFactory.createFailedDispatchQuery(session, query, Optional.empty(), throwable);
        queryCreated(failedDispatchQuery);
    }
}

Example 5 with QueryType

use of com.facebook.presto.spi.resourceGroups.QueryType in project presto by prestodb.

the class CoordinatorModule method setup.

@Override
protected void setup(Binder binder) {
    httpServerBinder(binder).bindResource("/ui", "webapp").withWelcomeFile("index.html");
    httpServerBinder(binder).bindResource("/tableau", "webapp/tableau");
    // discovery server
    install(installModuleIf(EmbeddedDiscoveryConfig.class, EmbeddedDiscoveryConfig::isEnabled, new EmbeddedDiscoveryModule()));
    // presto coordinator announcement
    discoveryBinder(binder).bindHttpAnnouncement("presto-coordinator");
    // statement resource
    jsonCodecBinder(binder).bindJsonCodec(QueryInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(QueryResults.class);
    jsonCodecBinder(binder).bindJsonCodec(SelectedRole.class);
    jaxrsBinder(binder).bind(QueuedStatementResource.class);
    jaxrsBinder(binder).bind(ExecutingStatementResource.class);
    binder.bind(StatementHttpExecutionMBean.class).in(Scopes.SINGLETON);
    newExporter(binder).export(StatementHttpExecutionMBean.class).withGeneratedName();
    // resource for serving static content
    jaxrsBinder(binder).bind(WebUiResource.class);
    // failure detector
    binder.install(new FailureDetectorModule());
    jaxrsBinder(binder).bind(NodeResource.class);
    jaxrsBinder(binder).bind(WorkerResource.class);
    httpClientBinder(binder).bindHttpClient("workerInfo", ForWorkerInfo.class);
    // query monitor
    jsonCodecBinder(binder).bindJsonCodec(OperatorInfo.class);
    configBinder(binder).bindConfig(QueryMonitorConfig.class);
    binder.bind(QueryMonitor.class).in(Scopes.SINGLETON);
    // query manager
    jaxrsBinder(binder).bind(QueryResource.class);
    jaxrsBinder(binder).bind(StageResource.class);
    jaxrsBinder(binder).bind(QueryStateInfoResource.class);
    jaxrsBinder(binder).bind(ResourceGroupStateInfoResource.class);
    binder.bind(QueryIdGenerator.class).in(Scopes.SINGLETON);
    binder.bind(QueryManager.class).to(SqlQueryManager.class).in(Scopes.SINGLETON);
    newExporter(binder).export(QueryManager.class).withGeneratedName();
    binder.bind(QueryPreparer.class).in(Scopes.SINGLETON);
    binder.bind(SessionSupplier.class).to(QuerySessionSupplier.class).in(Scopes.SINGLETON);
    binder.bind(InternalResourceGroupManager.class).in(Scopes.SINGLETON);
    newExporter(binder).export(InternalResourceGroupManager.class).withGeneratedName();
    binder.bind(ResourceGroupManager.class).to(InternalResourceGroupManager.class);
    binder.bind(LegacyResourceGroupConfigurationManager.class).in(Scopes.SINGLETON);
    binder.bind(RetryCircuitBreaker.class).in(Scopes.SINGLETON);
    newExporter(binder).export(RetryCircuitBreaker.class).withGeneratedName();
    binder.bind(LocalQueryProvider.class).in(Scopes.SINGLETON);
    jaxrsBinder(binder).bind(TaskInfoResource.class);
    // dispatcher
    binder.bind(DispatchManager.class).in(Scopes.SINGLETON);
    binder.bind(FailedDispatchQueryFactory.class).in(Scopes.SINGLETON);
    binder.bind(DispatchExecutor.class).in(Scopes.SINGLETON);
    // local dispatcher
    binder.bind(DispatchQueryFactory.class).to(LocalDispatchQueryFactory.class);
    // cluster memory manager
    binder.bind(ClusterMemoryManager.class).in(Scopes.SINGLETON);
    binder.bind(ClusterMemoryPoolManager.class).to(ClusterMemoryManager.class).in(Scopes.SINGLETON);
    httpClientBinder(binder).bindHttpClient("memoryManager", ForMemoryManager.class).withTracing().withConfigDefaults(config -> {
        config.setRequestTimeout(new Duration(10, SECONDS));
    });
    bindLowMemoryKiller(LowMemoryKillerPolicy.NONE, NoneLowMemoryKiller.class);
    bindLowMemoryKiller(LowMemoryKillerPolicy.TOTAL_RESERVATION, TotalReservationLowMemoryKiller.class);
    bindLowMemoryKiller(LowMemoryKillerPolicy.TOTAL_RESERVATION_ON_BLOCKED_NODES, TotalReservationOnBlockedNodesLowMemoryKiller.class);
    newExporter(binder).export(ClusterMemoryManager.class).withGeneratedName();
    // node monitor
    binder.bind(ClusterSizeMonitor.class).in(Scopes.SINGLETON);
    // statistics calculator
    binder.install(new StatsCalculatorModule());
    // cost calculator
    binder.bind(TaskCountEstimator.class).in(Scopes.SINGLETON);
    binder.bind(CostCalculator.class).to(CostCalculatorUsingExchanges.class).in(Scopes.SINGLETON);
    binder.bind(CostCalculator.class).annotatedWith(EstimatedExchanges.class).to(CostCalculatorWithEstimatedExchanges.class).in(Scopes.SINGLETON);
    binder.bind(CostComparator.class).in(Scopes.SINGLETON);
    // cluster statistics
    jaxrsBinder(binder).bind(ClusterStatsResource.class);
    // planner
    binder.bind(PlanFragmenter.class).in(Scopes.SINGLETON);
    binder.bind(PlanOptimizers.class).in(Scopes.SINGLETON);
    // query explainer
    binder.bind(QueryExplainer.class).in(Scopes.SINGLETON);
    // explain analyze
    binder.bind(ExplainAnalyzeContext.class).in(Scopes.SINGLETON);
    // execution scheduler
    binder.bind(RemoteTaskFactory.class).to(HttpRemoteTaskFactory.class).in(Scopes.SINGLETON);
    newExporter(binder).export(RemoteTaskFactory.class).withGeneratedName();
    binder.bind(RemoteTaskStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(RemoteTaskStats.class).withGeneratedName();
    httpClientBinder(binder).bindHttpClient("scheduler", ForScheduler.class).withTracing().withFilter(GenerateTraceTokenRequestFilter.class).withConfigDefaults(config -> {
        config.setRequestTimeout(new Duration(10, SECONDS));
        config.setMaxConnectionsPerServer(250);
    });
    binder.bind(ScheduledExecutorService.class).annotatedWith(ForScheduler.class).toInstance(newSingleThreadScheduledExecutor(threadsNamed("stage-scheduler")));
    // query execution
    binder.bind(ExecutorService.class).annotatedWith(ForQueryExecution.class).toInstance(newCachedThreadPool(threadsNamed("query-execution-%s")));
    binder.bind(QueryExecutionMBean.class).in(Scopes.SINGLETON);
    newExporter(binder).export(QueryExecutionMBean.class).as(generatedNameOf(QueryExecution.class));
    MapBinder<Class<? extends Statement>, QueryExecutionFactory<?>> executionBinder = newMapBinder(binder, new TypeLiteral<Class<? extends Statement>>() {
    }, new TypeLiteral<QueryExecution.QueryExecutionFactory<?>>() {
    });
    binder.bind(SplitSchedulerStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(SplitSchedulerStats.class).withGeneratedName();
    binder.bind(SqlQueryExecutionFactory.class).in(Scopes.SINGLETON);
    binder.bind(SectionExecutionFactory.class).in(Scopes.SINGLETON);
    Set<Map.Entry<Class<? extends Statement>, QueryType>> queryTypes = getAllQueryTypes().entrySet();
    // bind sql query statements to SqlQueryExecutionFactory
    queryTypes.stream().filter(entry -> entry.getValue() != QueryType.DATA_DEFINITION).forEach(entry -> executionBinder.addBinding(entry.getKey()).to(SqlQueryExecutionFactory.class).in(Scopes.SINGLETON));
    binder.bind(PartialResultQueryManager.class).in(Scopes.SINGLETON);
    // bind data definition statements to DataDefinitionExecutionFactory
    queryTypes.stream().filter(entry -> entry.getValue() == QueryType.DATA_DEFINITION && !isSessionTransactionControlStatement(entry.getKey())).forEach(entry -> executionBinder.addBinding(entry.getKey()).to(DDLDefinitionExecutionFactory.class).in(Scopes.SINGLETON));
    binder.bind(DDLDefinitionExecutionFactory.class).in(Scopes.SINGLETON);
    // bind session Control statements to SessionTransactionExecutionFactory
    queryTypes.stream().filter(entry -> (entry.getValue() == QueryType.DATA_DEFINITION && isSessionTransactionControlStatement(entry.getKey()))).forEach(entry -> executionBinder.addBinding(entry.getKey()).to(SessionDefinitionExecutionFactory.class).in(Scopes.SINGLETON));
    binder.bind(SessionDefinitionExecutionFactory.class).in(Scopes.SINGLETON);
    // helper class binding data definition tasks and statements
    PrestoDataDefBindingHelper.bindDDLDefinitionTasks(binder);
    PrestoDataDefBindingHelper.bindTransactionControlDefinitionTasks(binder);
    MapBinder<String, ExecutionPolicy> executionPolicyBinder = newMapBinder(binder, String.class, ExecutionPolicy.class);
    executionPolicyBinder.addBinding("all-at-once").to(AllAtOnceExecutionPolicy.class);
    executionPolicyBinder.addBinding("phased").to(PhasedExecutionPolicy.class);
    configBinder(binder).bindConfig(NodeResourceStatusConfig.class);
    binder.bind(NodeResourceStatusProvider.class).to(NodeResourceStatus.class).in(Scopes.SINGLETON);
    newOptionalBinder(binder, ResourceManagerProxy.class);
    install(installModuleIf(ServerConfig.class, ServerConfig::isResourceManagerEnabled, rmBinder -> {
        httpClientBinder(rmBinder).bindHttpClient("resourceManager", ForResourceManager.class);
        rmBinder.bind(ResourceManagerProxy.class).in(Scopes.SINGLETON);
    }));
    // cleanup
    binder.bind(ExecutorCleanup.class).in(Scopes.SINGLETON);
}