Examples with QueryStats com.facebook.presto.jdbc.QueryStats used on opensource projects

Example 1 with QueryStats

use of com.facebook.presto.jdbc.QueryStats in project presto by prestodb.

the class Validator method executeQuery.

private QueryResult executeQuery(String url, String username, String password, Query query, String sql, Duration timeout, Map<String, String> sessionProperties) {
    String queryId = null;
    try (Connection connection = DriverManager.getConnection(url, username, password)) {
        trySetConnectionProperties(query, connection);
        for (Map.Entry<String, String> entry : sessionProperties.entrySet()) {
            connection.unwrap(PrestoConnection.class).setSessionProperty(entry.getKey(), entry.getValue());
        }
        try (Statement statement = connection.createStatement()) {
            TimeLimiter limiter = new SimpleTimeLimiter();
            Stopwatch stopwatch = Stopwatch.createStarted();
            Statement limitedStatement = limiter.newProxy(statement, Statement.class, timeout.toMillis(), TimeUnit.MILLISECONDS);
            if (explainOnly) {
                sql = "EXPLAIN " + sql;
            }
            long start = System.nanoTime();
            PrestoStatement prestoStatement = limitedStatement.unwrap(PrestoStatement.class);
            ProgressMonitor progressMonitor = new ProgressMonitor();
            prestoStatement.setProgressMonitor(progressMonitor);
            try {
                boolean isSelectQuery = limitedStatement.execute(sql);
                List<List<Object>> results = null;
                if (isSelectQuery) {
                    results = limiter.callWithTimeout(getResultSetConverter(limitedStatement.getResultSet()), timeout.toMillis() - stopwatch.elapsed(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS, true);
                } else {
                    results = ImmutableList.of(ImmutableList.of(limitedStatement.getLargeUpdateCount()));
                }
                prestoStatement.clearProgressMonitor();
                QueryStats queryStats = progressMonitor.getFinalQueryStats();
                if (queryStats == null) {
                    throw new VerifierException("Cannot fetch query stats");
                }
                Duration queryCpuTime = new Duration(queryStats.getCpuTimeMillis(), TimeUnit.MILLISECONDS);
                queryId = queryStats.getQueryId();
                return new QueryResult(State.SUCCESS, null, nanosSince(start), queryCpuTime, queryId, results);
            } catch (AssertionError e) {
                if (e.getMessage().startsWith("unimplemented type:")) {
                    return new QueryResult(State.INVALID, null, null, null, queryId, ImmutableList.of());
                }
                throw e;
            } catch (SQLException | VerifierException e) {
                throw e;
            } catch (UncheckedTimeoutException e) {
                return new QueryResult(State.TIMEOUT, null, null, null, queryId, ImmutableList.of());
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw Throwables.propagate(e);
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    } catch (SQLException e) {
        Exception exception = e;
        if (("Error executing query".equals(e.getMessage()) || "Error fetching results".equals(e.getMessage())) && (e.getCause() instanceof Exception)) {
            exception = (Exception) e.getCause();
        }
        State state = isPrestoQueryInvalid(e) ? State.INVALID : State.FAILED;
        return new QueryResult(state, exception, null, null, null, null);
    } catch (VerifierException e) {
        return new QueryResult(State.TOO_MANY_ROWS, e, null, null, null, null);
    }
}

Example 2 with QueryStats

use of com.facebook.presto.jdbc.QueryStats in project presto by prestodb.

the class DeterminismAnalyzer method analyze.

private DeterminismAnalysis analyze(QueryObjectBundle control, ChecksumResult controlChecksum, DeterminismAnalysisDetails.Builder determinismAnalysisDetails) {
    // Handle mutable catalogs
    if (isNonDeterministicCatalogReferenced(control.getQuery())) {
        return NON_DETERMINISTIC_CATALOG;
    }
    // Handle limit query
    LimitQueryDeterminismAnalysis limitQueryAnalysis = new LimitQueryDeterminismAnalyzer(prestoAction, handleLimitQuery, control.getQuery(), controlChecksum.getRowCount(), determinismAnalysisDetails).analyze();
    switch(limitQueryAnalysis) {
        case NOT_RUN:
        case FAILED_QUERY_FAILURE:
        case DETERMINISTIC:
            // try the next analysis
            break;
        case NON_DETERMINISTIC:
            return NON_DETERMINISTIC_LIMIT_CLAUSE;
        case FAILED_DATA_CHANGED:
            return ANALYSIS_FAILED_DATA_CHANGED;
        default:
            throw new IllegalArgumentException(format("Invalid limitQueryAnalysis: %s", limitQueryAnalysis));
    }
    // Rerun control query multiple times
    List<Column> columns = getColumns(prestoAction, typeManager, control.getObjectName());
    Map<QueryBundle, DeterminismAnalysisRun.Builder> queryRuns = new HashMap<>();
    try {
        for (int i = 0; i < maxAnalysisRuns; i++) {
            QueryObjectBundle queryBundle = queryRewriter.rewriteQuery(sourceQuery.getQuery(CONTROL), CONTROL);
            DeterminismAnalysisRun.Builder run = determinismAnalysisDetails.addRun().setTableName(queryBundle.getObjectName().toString());
            queryRuns.put(queryBundle, run);
            // Rerun setup and main query
            queryBundle.getSetupQueries().forEach(query -> runAndConsume(() -> prestoAction.execute(query, DETERMINISM_ANALYSIS_SETUP), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::addSetupQueryId)));
            runAndConsume(() -> prestoAction.execute(queryBundle.getQuery(), DETERMINISM_ANALYSIS_MAIN), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setQueryId));
            // Run checksum query
            Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns);
            ChecksumResult testChecksum = getOnlyElement(callAndConsume(() -> prestoAction.execute(checksumQuery, DETERMINISM_ANALYSIS_CHECKSUM, ChecksumResult::fromResultSet), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setChecksumQueryId)).getResults());
            DeterminismAnalysis analysis = matchResultToDeterminism(match(checksumValidator, columns, columns, controlChecksum, testChecksum));
            if (analysis != DETERMINISTIC) {
                return analysis;
            }
        }
        return DETERMINISTIC;
    } catch (QueryException qe) {
        return ANALYSIS_FAILED_QUERY_FAILURE;
    } finally {
        if (runTeardown) {
            queryRuns.forEach((queryBundle, run) -> teardownSafely(prestoAction, Optional.of(queryBundle), queryStats -> queryStats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::addTeardownQueryId)));
        }
    }
}

Example 3 with QueryStats

use of com.facebook.presto.jdbc.QueryStats in project presto by prestodb.

the class LimitQueryDeterminismAnalyzer method analyzeLimitNoOrderBy.

private LimitQueryDeterminismAnalysis analyzeLimitNoOrderBy(Query newLimitQuery, long limit) {
    Query rowCountQuery = simpleQuery(new Select(false, ImmutableList.of(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of(new LongLiteral("1")))))), new TableSubquery(newLimitQuery));
    QueryResult<Long> result = callAndConsume(() -> prestoAction.execute(rowCountQuery, DETERMINISM_ANALYSIS_MAIN, resultSet -> Optional.of(resultSet.getLong(1))), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(determinismAnalysisDetails::setLimitQueryAnalysisQueryId));
    long rowCountHigherLimit = getOnlyElement(result.getResults());
    if (rowCountHigherLimit == rowCount) {
        return DETERMINISTIC;
    }
    if (rowCountHigherLimit > rowCount) {
        return NON_DETERMINISTIC;
    }
    return FAILED_DATA_CHANGED;
}

Example 4 with QueryStats

use of com.facebook.presto.jdbc.QueryStats in project presto by prestodb.

the class AbstractPhaseExecutor method runQuery.

protected BenchmarkQueryEvent runQuery(BenchmarkQuery benchmarkQuery) {
    Optional<QueryStats> queryStats = Optional.empty();
    Statement statement = sqlParser.createStatement(benchmarkQuery.getQuery(), parsingOptions);
    PrestoAction prestoAction = prestoActionFactory.get(benchmarkQuery);
    try {
        queryStats = Optional.of(prestoAction.execute(statement));
        return buildEvent(benchmarkQuery, queryStats, Optional.empty());
    } catch (QueryException e) {
        return buildEvent(benchmarkQuery, queryStats, Optional.of(e));
    } catch (Throwable t) {
        log.error(t);
        return buildEvent(benchmarkQuery, queryStats, Optional.empty());
    }
}

Example 5 with QueryStats

use of com.facebook.presto.jdbc.QueryStats in project presto by prestodb.

the class TooManyOpenPartitionsFailureResolver method resolveQueryFailure.

@Override
public Optional<String> resolveQueryFailure(QueryStats controlQueryStats, QueryException queryException, Optional<QueryObjectBundle> test) {
    if (!test.isPresent()) {
        return Optional.empty();
    }
    return mapMatchingPrestoException(queryException, TEST_MAIN, ImmutableSet.of(HIVE_TOO_MANY_OPEN_PARTITIONS), e -> {
        try {
            ShowCreate showCreate = new ShowCreate(TABLE, test.get().getObjectName());
            String showCreateResult = getOnlyElement(prestoAction.execute(showCreate, DESCRIBE, resultSet -> Optional.of(resultSet.getString(1))).getResults());
            CreateTable createTable = (CreateTable) sqlParser.createStatement(showCreateResult, ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build());
            List<Property> bucketCountProperty = createTable.getProperties().stream().filter(property -> property.getName().getValue().equals(BUCKET_COUNT_PROPERTY)).collect(toImmutableList());
            if (bucketCountProperty.size() != 1) {
                return Optional.empty();
            }
            long bucketCount = ((LongLiteral) getOnlyElement(bucketCountProperty).getValue()).getValue();
            int testClusterSize = this.testClusterSizeSupplier.get();
            if (testClusterSize * maxBucketPerWriter < bucketCount) {
                return Optional.of("Not enough workers on test cluster");
            }
            return Optional.empty();
        } catch (Throwable t) {
            log.warn(t, "Exception when resolving HIVE_TOO_MANY_OPEN_PARTITIONS");
            return Optional.empty();
        }
    });
}