Search in sources :

Example 1 with DETERMINISM_ANALYSIS_SETUP

use of com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS_SETUP in project presto by prestodb.

the class DeterminismAnalyzer method analyze.

private DeterminismAnalysis analyze(QueryObjectBundle control, ChecksumResult controlChecksum, DeterminismAnalysisDetails.Builder determinismAnalysisDetails) {
    // Handle mutable catalogs
    if (isNonDeterministicCatalogReferenced(control.getQuery())) {
        return NON_DETERMINISTIC_CATALOG;
    }
    // Handle limit query
    LimitQueryDeterminismAnalysis limitQueryAnalysis = new LimitQueryDeterminismAnalyzer(prestoAction, handleLimitQuery, control.getQuery(), controlChecksum.getRowCount(), determinismAnalysisDetails).analyze();
    switch(limitQueryAnalysis) {
        case NOT_RUN:
        case FAILED_QUERY_FAILURE:
        case DETERMINISTIC:
            // try the next analysis
            break;
        case NON_DETERMINISTIC:
            return NON_DETERMINISTIC_LIMIT_CLAUSE;
        case FAILED_DATA_CHANGED:
            return ANALYSIS_FAILED_DATA_CHANGED;
        default:
            throw new IllegalArgumentException(format("Invalid limitQueryAnalysis: %s", limitQueryAnalysis));
    }
    // Rerun control query multiple times
    List<Column> columns = getColumns(prestoAction, typeManager, control.getObjectName());
    Map<QueryBundle, DeterminismAnalysisRun.Builder> queryRuns = new HashMap<>();
    try {
        for (int i = 0; i < maxAnalysisRuns; i++) {
            QueryObjectBundle queryBundle = queryRewriter.rewriteQuery(sourceQuery.getQuery(CONTROL), CONTROL);
            DeterminismAnalysisRun.Builder run = determinismAnalysisDetails.addRun().setTableName(queryBundle.getObjectName().toString());
            queryRuns.put(queryBundle, run);
            // Rerun setup and main query
            queryBundle.getSetupQueries().forEach(query -> runAndConsume(() -> prestoAction.execute(query, DETERMINISM_ANALYSIS_SETUP), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::addSetupQueryId)));
            runAndConsume(() -> prestoAction.execute(queryBundle.getQuery(), DETERMINISM_ANALYSIS_MAIN), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setQueryId));
            // Run checksum query
            Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns);
            ChecksumResult testChecksum = getOnlyElement(callAndConsume(() -> prestoAction.execute(checksumQuery, DETERMINISM_ANALYSIS_CHECKSUM, ChecksumResult::fromResultSet), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setChecksumQueryId)).getResults());
            DeterminismAnalysis analysis = matchResultToDeterminism(match(checksumValidator, columns, columns, controlChecksum, testChecksum));
            if (analysis != DETERMINISTIC) {
                return analysis;
            }
        }
        return DETERMINISTIC;
    } catch (QueryException qe) {
        return ANALYSIS_FAILED_QUERY_FAILURE;
    } finally {
        if (runTeardown) {
            queryRuns.forEach((queryBundle, run) -> teardownSafely(prestoAction, Optional.of(queryBundle), queryStats -> queryStats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::addTeardownQueryId)));
        }
    }
}
Also used : NON_DETERMINISTIC_ROW_COUNT(com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_ROW_COUNT) Table(com.facebook.presto.sql.tree.Table) DataVerificationUtil.match(com.facebook.presto.verifier.framework.DataVerificationUtil.match) ANALYSIS_FAILED_DATA_CHANGED(com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_DATA_CHANGED) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) DETERMINISTIC(com.facebook.presto.verifier.framework.DeterminismAnalysis.DETERMINISTIC) VerifierUtil.callAndConsume(com.facebook.presto.verifier.framework.VerifierUtil.callAndConsume) NON_DETERMINISTIC_CATALOG(com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_CATALOG) NON_DETERMINISTIC_COLUMNS(com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_COLUMNS) DataVerificationUtil.getColumns(com.facebook.presto.verifier.framework.DataVerificationUtil.getColumns) TypeManager(com.facebook.presto.common.type.TypeManager) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) QueryStats(com.facebook.presto.jdbc.QueryStats) ANALYSIS_FAILED_INCONSISTENT_SCHEMA(com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_INCONSISTENT_SCHEMA) QueryRewriter(com.facebook.presto.verifier.rewrite.QueryRewriter) DETERMINISM_ANALYSIS_SETUP(com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS_SETUP) DeterminismAnalysisRun(com.facebook.presto.verifier.event.DeterminismAnalysisRun) ANALYSIS_FAILED_QUERY_FAILURE(com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_QUERY_FAILURE) DETERMINISM_ANALYSIS_CHECKSUM(com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS_CHECKSUM) ImmutableSet(com.google.common.collect.ImmutableSet) DataVerificationUtil.teardownSafely(com.facebook.presto.verifier.framework.DataVerificationUtil.teardownSafely) Query(com.facebook.presto.sql.tree.Query) Node(com.facebook.presto.sql.tree.Node) AstVisitor(com.facebook.presto.sql.tree.AstVisitor) Set(java.util.Set) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) CONTROL(com.facebook.presto.verifier.framework.ClusterType.CONTROL) String.format(java.lang.String.format) ChecksumResult(com.facebook.presto.verifier.checksum.ChecksumResult) DeterminismAnalysisDetails(com.facebook.presto.verifier.event.DeterminismAnalysisDetails) List(java.util.List) VerifierUtil.runAndConsume(com.facebook.presto.verifier.framework.VerifierUtil.runAndConsume) ChecksumValidator(com.facebook.presto.verifier.checksum.ChecksumValidator) PrestoAction(com.facebook.presto.verifier.prestoaction.PrestoAction) Optional(java.util.Optional) VisibleForTesting(com.google.common.annotations.VisibleForTesting) NON_DETERMINISTIC_LIMIT_CLAUSE(com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_LIMIT_CLAUSE) DETERMINISM_ANALYSIS_MAIN(com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS_MAIN) Statement(com.facebook.presto.sql.tree.Statement) ChecksumResult(com.facebook.presto.verifier.checksum.ChecksumResult) Query(com.facebook.presto.sql.tree.Query) HashMap(java.util.HashMap) QueryStats(com.facebook.presto.jdbc.QueryStats) DeterminismAnalysisRun(com.facebook.presto.verifier.event.DeterminismAnalysisRun)

Aggregations

TypeManager (com.facebook.presto.common.type.TypeManager)1 QueryStats (com.facebook.presto.jdbc.QueryStats)1 AstVisitor (com.facebook.presto.sql.tree.AstVisitor)1 Node (com.facebook.presto.sql.tree.Node)1 Query (com.facebook.presto.sql.tree.Query)1 Statement (com.facebook.presto.sql.tree.Statement)1 Table (com.facebook.presto.sql.tree.Table)1 ChecksumResult (com.facebook.presto.verifier.checksum.ChecksumResult)1 ChecksumValidator (com.facebook.presto.verifier.checksum.ChecksumValidator)1 DeterminismAnalysisDetails (com.facebook.presto.verifier.event.DeterminismAnalysisDetails)1 DeterminismAnalysisRun (com.facebook.presto.verifier.event.DeterminismAnalysisRun)1 CONTROL (com.facebook.presto.verifier.framework.ClusterType.CONTROL)1 DataVerificationUtil.getColumns (com.facebook.presto.verifier.framework.DataVerificationUtil.getColumns)1 DataVerificationUtil.match (com.facebook.presto.verifier.framework.DataVerificationUtil.match)1 DataVerificationUtil.teardownSafely (com.facebook.presto.verifier.framework.DataVerificationUtil.teardownSafely)1 ANALYSIS_FAILED_DATA_CHANGED (com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_DATA_CHANGED)1 ANALYSIS_FAILED_INCONSISTENT_SCHEMA (com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_INCONSISTENT_SCHEMA)1 ANALYSIS_FAILED_QUERY_FAILURE (com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_QUERY_FAILURE)1 DETERMINISTIC (com.facebook.presto.verifier.framework.DeterminismAnalysis.DETERMINISTIC)1 NON_DETERMINISTIC_CATALOG (com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_CATALOG)1