Examples with PreparedQuery com.facebook.presto.execution.QueryPreparer.PreparedQuery used on opensource projects

Example 6 with PreparedQuery

use of com.facebook.presto.execution.QueryPreparer.PreparedQuery in project presto by prestodb.

the class DispatchManager method createQueryInternal.

/**
 *  Creates and registers a dispatch query with the query tracker.  This method will never fail to register a query with the query
 *  tracker.  If an error occurs while, creating a dispatch query a failed dispatch will be created and registered.
 */
private <C> void createQueryInternal(QueryId queryId, String slug, int retryCount, SessionContext sessionContext, String query, ResourceGroupManager<C> resourceGroupManager) {
    Session session = null;
    PreparedQuery preparedQuery;
    try {
        if (query.length() > maxQueryLength) {
            int queryLength = query.length();
            query = query.substring(0, maxQueryLength);
            throw new PrestoException(QUERY_TEXT_TOO_LARGE, format("Query text length (%s) exceeds the maximum length (%s)", queryLength, maxQueryLength));
        }
        // decode session
        session = sessionSupplier.createSession(queryId, sessionContext);
        // prepare query
        WarningCollector warningCollector = warningCollectorFactory.create(getWarningHandlingLevel(session));
        preparedQuery = queryPreparer.prepareQuery(session, query, warningCollector);
        query = preparedQuery.getFormattedQuery().orElse(query);
        // select resource group
        Optional<QueryType> queryType = getQueryType(preparedQuery.getStatement().getClass());
        SelectionContext<C> selectionContext = resourceGroupManager.selectGroup(new SelectionCriteria(sessionContext.getIdentity().getPrincipal().isPresent(), sessionContext.getIdentity().getUser(), Optional.ofNullable(sessionContext.getSource()), sessionContext.getClientTags(), sessionContext.getResourceEstimates(), queryType.map(Enum::name)));
        // apply system default session properties (does not override user set properties)
        session = sessionPropertyDefaults.newSessionWithDefaultProperties(session, queryType.map(Enum::name), Optional.of(selectionContext.getResourceGroupId()));
        // mark existing transaction as active
        transactionManager.activateTransaction(session, isTransactionControlStatement(preparedQuery.getStatement()), accessControl);
        DispatchQuery dispatchQuery = dispatchQueryFactory.createDispatchQuery(session, query, preparedQuery, slug, retryCount, selectionContext.getResourceGroupId(), queryType, warningCollector, (dq) -> resourceGroupManager.submit(preparedQuery.getStatement(), dq, selectionContext, queryExecutor));
        boolean queryAdded = queryCreated(dispatchQuery);
        if (queryAdded && !dispatchQuery.isDone()) {
            try {
                clusterStatusSender.registerQuery(dispatchQuery);
                dispatchQuery.startWaitingForPrerequisites();
            } catch (Throwable e) {
                // dispatch query has already been registered, so just fail it directly
                dispatchQuery.fail(e);
            }
        }
    } catch (Throwable throwable) {
        // creation must never fail, so register a failed query in this case
        if (session == null) {
            session = Session.builder(new SessionPropertyManager()).setQueryId(queryId).setIdentity(sessionContext.getIdentity()).setSource(sessionContext.getSource()).build();
        }
        DispatchQuery failedDispatchQuery = failedDispatchQueryFactory.createFailedDispatchQuery(session, query, Optional.empty(), throwable);
        queryCreated(failedDispatchQuery);
    }
}

Example 7 with PreparedQuery

use of com.facebook.presto.execution.QueryPreparer.PreparedQuery in project presto by prestodb.

the class PrestoSparkQueryExecutionFactory method create.

@Override
public IPrestoSparkQueryExecution create(SparkContext sparkContext, PrestoSparkSession prestoSparkSession, Optional<String> sqlText, Optional<String> sqlLocation, Optional<String> sqlFileHexHash, Optional<String> sqlFileSizeInBytes, Optional<String> sparkQueueName, PrestoSparkTaskExecutorFactoryProvider executorFactoryProvider, Optional<String> queryStatusInfoOutputLocation, Optional<String> queryDataOutputLocation) {
    PrestoSparkConfInitializer.checkInitialized(sparkContext);
    String sql;
    if (sqlText.isPresent()) {
        checkArgument(!sqlLocation.isPresent(), "sqlText and sqlLocation should not be set at the same time");
        sql = sqlText.get();
    } else {
        checkArgument(sqlLocation.isPresent(), "sqlText or sqlLocation must be present");
        byte[] sqlFileBytes = metadataStorage.read(sqlLocation.get());
        if (sqlFileSizeInBytes.isPresent()) {
            if (Integer.valueOf(sqlFileSizeInBytes.get()) != sqlFileBytes.length) {
                throw new PrestoException(MALFORMED_QUERY_FILE, format("sql file size %s is different from expected sqlFileSizeInBytes %s", sqlFileBytes.length, sqlFileSizeInBytes.get()));
            }
        }
        if (sqlFileHexHash.isPresent()) {
            try {
                MessageDigest md = MessageDigest.getInstance("SHA-512");
                String actualHexHashCode = BaseEncoding.base16().lowerCase().encode(md.digest(sqlFileBytes));
                if (!sqlFileHexHash.get().equals(actualHexHashCode)) {
                    throw new PrestoException(MALFORMED_QUERY_FILE, format("actual hash code %s is different from expected sqlFileHexHash %s", actualHexHashCode, sqlFileHexHash.get()));
                }
            } catch (NoSuchAlgorithmException e) {
                throw new PrestoException(GENERIC_INTERNAL_ERROR, "unsupported hash algorithm", e);
            }
        }
        sql = new String(sqlFileBytes, UTF_8);
    }
    log.info("Query: %s", sql);
    QueryStateTimer queryStateTimer = new QueryStateTimer(systemTicker());
    queryStateTimer.beginPlanning();
    QueryId queryId = queryIdGenerator.createNextQueryId();
    log.info("Starting execution for presto query: %s", queryId);
    System.out.printf("Query id: %s\n", queryId);
    sparkContext.conf().set(PRESTO_QUERY_ID_CONFIG, queryId.getId());
    SessionContext sessionContext = PrestoSparkSessionContext.createFromSessionInfo(prestoSparkSession, credentialsProviders, authenticatorProviders);
    Session session = sessionSupplier.createSession(queryId, sessionContext);
    session = sessionPropertyDefaults.newSessionWithDefaultProperties(session, Optional.empty(), Optional.empty());
    WarningCollector warningCollector = warningCollectorFactory.create(getWarningHandlingLevel(session));
    PlanAndMore planAndMore = null;
    try {
        TransactionId transactionId = transactionManager.beginTransaction(true);
        session = session.beginTransactionId(transactionId, transactionManager, accessControl);
        queryMonitor.queryCreatedEvent(new BasicQueryInfo(createQueryInfo(session, sql, PLANNING, Optional.empty(), sparkQueueName, Optional.empty(), queryStateTimer, Optional.empty(), warningCollector)));
        // including queueing time
        Duration queryMaxRunTime = getQueryMaxRunTime(session);
        // excluding queueing time
        Duration queryMaxExecutionTime = getQueryMaxExecutionTime(session);
        // pick a smaller one as we are not tracking queueing for Presto on Spark
        Duration queryTimeout = queryMaxRunTime.compareTo(queryMaxExecutionTime) < 0 ? queryMaxRunTime : queryMaxExecutionTime;
        long queryCompletionDeadline = System.currentTimeMillis() + queryTimeout.toMillis();
        settingsRequirements.verify(sparkContext, session);
        queryStateTimer.beginAnalyzing();
        PreparedQuery preparedQuery = queryPreparer.prepareQuery(session, sql, warningCollector);
        Optional<QueryType> queryType = StatementUtils.getQueryType(preparedQuery.getStatement().getClass());
        if (queryType.isPresent() && (queryType.get() == QueryType.DATA_DEFINITION)) {
            queryStateTimer.endAnalysis();
            DDLDefinitionTask<?> task = (DDLDefinitionTask<?>) ddlTasks.get(preparedQuery.getStatement().getClass());
            return new PrestoSparkDataDefinitionExecution(task, preparedQuery.getStatement(), transactionManager, accessControl, metadata, session, queryStateTimer, warningCollector);
        } else {
            planAndMore = queryPlanner.createQueryPlan(session, preparedQuery, warningCollector);
            SubPlan fragmentedPlan = planFragmenter.fragmentQueryPlan(session, planAndMore.getPlan(), warningCollector);
            log.info(textDistributedPlan(fragmentedPlan, metadata.getFunctionAndTypeManager(), session, true));
            fragmentedPlan = configureOutputPartitioning(session, fragmentedPlan);
            TableWriteInfo tableWriteInfo = getTableWriteInfo(session, fragmentedPlan);
            JavaSparkContext javaSparkContext = new JavaSparkContext(sparkContext);
            CollectionAccumulator<SerializedTaskInfo> taskInfoCollector = new CollectionAccumulator<>();
            taskInfoCollector.register(sparkContext, Option.empty(), false);
            CollectionAccumulator<PrestoSparkShuffleStats> shuffleStatsCollector = new CollectionAccumulator<>();
            shuffleStatsCollector.register(sparkContext, Option.empty(), false);
            TempStorage tempStorage = tempStorageManager.getTempStorage(storageBasedBroadcastJoinStorage);
            queryStateTimer.endAnalysis();
            return new PrestoSparkQueryExecution(javaSparkContext, session, queryMonitor, taskInfoCollector, shuffleStatsCollector, prestoSparkTaskExecutorFactory, executorFactoryProvider, queryStateTimer, warningCollector, sql, planAndMore, fragmentedPlan, sparkQueueName, taskInfoCodec, sparkTaskDescriptorJsonCodec, queryStatusInfoJsonCodec, queryDataJsonCodec, rddFactory, tableWriteInfo, transactionManager, createPagesSerde(blockEncodingManager), executionExceptionFactory, queryTimeout, queryCompletionDeadline, metadataStorage, queryStatusInfoOutputLocation, queryDataOutputLocation, tempStorage, nodeMemoryConfig, waitTimeMetrics);
        }
    } catch (Throwable executionFailure) {
        queryStateTimer.beginFinishing();
        try {
            rollback(session, transactionManager);
        } catch (RuntimeException rollbackFailure) {
            log.error(rollbackFailure, "Encountered error when performing rollback");
        }
        queryStateTimer.endQuery();
        Optional<ExecutionFailureInfo> failureInfo = Optional.empty();
        if (executionFailure instanceof PrestoSparkExecutionException) {
            failureInfo = executionExceptionFactory.extractExecutionFailureInfo((PrestoSparkExecutionException) executionFailure);
            verify(failureInfo.isPresent());
        }
        if (!failureInfo.isPresent()) {
            failureInfo = Optional.of(toFailure(executionFailure));
        }
        try {
            QueryInfo queryInfo = createQueryInfo(session, sql, FAILED, Optional.ofNullable(planAndMore), sparkQueueName, failureInfo, queryStateTimer, Optional.empty(), warningCollector);
            queryMonitor.queryCompletedEvent(queryInfo);
            if (queryStatusInfoOutputLocation.isPresent()) {
                PrestoSparkQueryStatusInfo prestoSparkQueryStatusInfo = createPrestoSparkQueryInfo(queryInfo, Optional.ofNullable(planAndMore), warningCollector, OptionalLong.empty());
                metadataStorage.write(queryStatusInfoOutputLocation.get(), queryStatusInfoJsonCodec.toJsonBytes(prestoSparkQueryStatusInfo));
            }
        } catch (RuntimeException eventFailure) {
            log.error(eventFailure, "Error publishing query immediate failure event");
        }
        throw failureInfo.get().toFailure();
    }
}