Examples with ChecksumValidator com.facebook.presto.verifier.checksum.ChecksumValidator used on opensource projects

Example 1 with ChecksumValidator

use of com.facebook.presto.verifier.checksum.ChecksumValidator in project presto by prestodb.

the class DeterminismAnalyzer method analyze.

private DeterminismAnalysis analyze(QueryObjectBundle control, ChecksumResult controlChecksum, DeterminismAnalysisDetails.Builder determinismAnalysisDetails) {
    // Handle mutable catalogs
    if (isNonDeterministicCatalogReferenced(control.getQuery())) {
        return NON_DETERMINISTIC_CATALOG;
    }
    // Handle limit query
    LimitQueryDeterminismAnalysis limitQueryAnalysis = new LimitQueryDeterminismAnalyzer(prestoAction, handleLimitQuery, control.getQuery(), controlChecksum.getRowCount(), determinismAnalysisDetails).analyze();
    switch(limitQueryAnalysis) {
        case NOT_RUN:
        case FAILED_QUERY_FAILURE:
        case DETERMINISTIC:
            // try the next analysis
            break;
        case NON_DETERMINISTIC:
            return NON_DETERMINISTIC_LIMIT_CLAUSE;
        case FAILED_DATA_CHANGED:
            return ANALYSIS_FAILED_DATA_CHANGED;
        default:
            throw new IllegalArgumentException(format("Invalid limitQueryAnalysis: %s", limitQueryAnalysis));
    }
    // Rerun control query multiple times
    List<Column> columns = getColumns(prestoAction, typeManager, control.getObjectName());
    Map<QueryBundle, DeterminismAnalysisRun.Builder> queryRuns = new HashMap<>();
    try {
        for (int i = 0; i < maxAnalysisRuns; i++) {
            QueryObjectBundle queryBundle = queryRewriter.rewriteQuery(sourceQuery.getQuery(CONTROL), CONTROL);
            DeterminismAnalysisRun.Builder run = determinismAnalysisDetails.addRun().setTableName(queryBundle.getObjectName().toString());
            queryRuns.put(queryBundle, run);
            // Rerun setup and main query
            queryBundle.getSetupQueries().forEach(query -> runAndConsume(() -> prestoAction.execute(query, DETERMINISM_ANALYSIS_SETUP), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::addSetupQueryId)));
            runAndConsume(() -> prestoAction.execute(queryBundle.getQuery(), DETERMINISM_ANALYSIS_MAIN), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setQueryId));
            // Run checksum query
            Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns);
            ChecksumResult testChecksum = getOnlyElement(callAndConsume(() -> prestoAction.execute(checksumQuery, DETERMINISM_ANALYSIS_CHECKSUM, ChecksumResult::fromResultSet), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setChecksumQueryId)).getResults());
            DeterminismAnalysis analysis = matchResultToDeterminism(match(checksumValidator, columns, columns, controlChecksum, testChecksum));
            if (analysis != DETERMINISTIC) {
                return analysis;
            }
        }
        return DETERMINISTIC;
    } catch (QueryException qe) {
        return ANALYSIS_FAILED_QUERY_FAILURE;
    } finally {
        if (runTeardown) {
            queryRuns.forEach((queryBundle, run) -> teardownSafely(prestoAction, Optional.of(queryBundle), queryStats -> queryStats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::addTeardownQueryId)));
        }
    }
}

Example 2 with ChecksumValidator

use of com.facebook.presto.verifier.checksum.ChecksumValidator in project presto by prestodb.

the class TestDeterminismAnalyzer method createDeterminismAnalyzer.

private static DeterminismAnalyzer createDeterminismAnalyzer(String mutableCatalogPattern) {
    QueryConfiguration configuration = new QueryConfiguration(CATALOG, SCHEMA, Optional.of("user"), Optional.empty(), Optional.empty());
    VerificationContext verificationContext = VerificationContext.create(SUITE, NAME);
    VerifierConfig verifierConfig = new VerifierConfig().setTestId("test-id");
    RetryConfig retryConfig = new RetryConfig();
    QueryActionsConfig queryActionsConfig = new QueryActionsConfig();
    TypeManager typeManager = createTypeManager();
    PrestoAction prestoAction = new JdbcPrestoAction(PrestoExceptionClassifier.defaultBuilder().build(), configuration, verificationContext, new JdbcUrlSelector(ImmutableList.of()), new PrestoActionConfig(), queryActionsConfig.getMetadataTimeout(), queryActionsConfig.getChecksumTimeout(), retryConfig, retryConfig, verifierConfig);
    QueryRewriter queryRewriter = new QueryRewriter(sqlParser, typeManager, prestoAction, ImmutableMap.of(CONTROL, QualifiedName.of("tmp_verifier_c"), TEST, QualifiedName.of("tmp_verifier_t")), ImmutableMap.of());
    ChecksumValidator checksumValidator = createChecksumValidator(verifierConfig);
    SourceQuery sourceQuery = new SourceQuery("test", "", "", "", configuration, configuration);
    return new DeterminismAnalyzer(sourceQuery, prestoAction, queryRewriter, checksumValidator, typeManager, new DeterminismAnalyzerConfig().setNonDeterministicCatalogs(mutableCatalogPattern));
}

Example 3 with ChecksumValidator

use of com.facebook.presto.verifier.checksum.ChecksumValidator in project presto by prestodb.

the class VerifierTestUtil method createChecksumValidator.

public static ChecksumValidator createChecksumValidator(VerifierConfig verifierConfig) {
    Map<Column.Category, Provider<ColumnValidator>> lazyValidators = new HashMap<>();
    Map<Column.Category, Provider<ColumnValidator>> validators = ImmutableMap.of(Column.Category.SIMPLE, SimpleColumnValidator::new, Column.Category.FLOATING_POINT, () -> new FloatingPointColumnValidator(verifierConfig), Column.Category.ARRAY, ArrayColumnValidator::new, Column.Category.ROW, () -> new RowColumnValidator(lazyValidators), Column.Category.MAP, MapColumnValidator::new);
    lazyValidators.putAll(validators);
    return new ChecksumValidator(validators);
}