Search in sources :

Example 6 with Query

use of com.facebook.presto.sql.tree.Query in project presto by prestodb.

the class TestSqlParser method testLateral.

@Test
public void testLateral() {
    Lateral lateralRelation = new Lateral(new Query(Optional.empty(), new Values(ImmutableList.of(new LongLiteral("1"))), Optional.empty(), Optional.empty(), Optional.empty()));
    assertStatement("SELECT * FROM t, LATERAL (VALUES 1) a(x)", simpleQuery(selectList(new AllColumns()), new Join(Join.Type.IMPLICIT, new Table(QualifiedName.of("t")), new AliasedRelation(lateralRelation, identifier("a"), ImmutableList.of(identifier("x"))), Optional.empty())));
    assertStatement("SELECT * FROM t CROSS JOIN LATERAL (VALUES 1) ", simpleQuery(selectList(new AllColumns()), new Join(Join.Type.CROSS, new Table(QualifiedName.of("t")), lateralRelation, Optional.empty())));
    assertStatement("SELECT * FROM t FULL JOIN LATERAL (VALUES 1) ON true", simpleQuery(selectList(new AllColumns()), new Join(Join.Type.FULL, new Table(QualifiedName.of("t")), lateralRelation, Optional.of(new JoinOn(BooleanLiteral.TRUE_LITERAL)))));
}
Also used : CreateTable(com.facebook.presto.sql.tree.CreateTable) DropTable(com.facebook.presto.sql.tree.DropTable) Table(com.facebook.presto.sql.tree.Table) RenameTable(com.facebook.presto.sql.tree.RenameTable) Query(com.facebook.presto.sql.tree.Query) QueryUtil.simpleQuery(com.facebook.presto.sql.QueryUtil.simpleQuery) WithQuery(com.facebook.presto.sql.tree.WithQuery) LongLiteral(com.facebook.presto.sql.tree.LongLiteral) Lateral(com.facebook.presto.sql.tree.Lateral) Values(com.facebook.presto.sql.tree.Values) NaturalJoin(com.facebook.presto.sql.tree.NaturalJoin) Join(com.facebook.presto.sql.tree.Join) AllColumns(com.facebook.presto.sql.tree.AllColumns) JoinOn(com.facebook.presto.sql.tree.JoinOn) AliasedRelation(com.facebook.presto.sql.tree.AliasedRelation) Test(org.testng.annotations.Test)

Example 7 with Query

use of com.facebook.presto.sql.tree.Query in project presto by prestodb.

the class TestSqlParser method testWith.

@Test
public void testWith() {
    assertStatement("WITH a (t, u) AS (SELECT * FROM x), b AS (SELECT * FROM y) TABLE z", new Query(Optional.of(new With(false, ImmutableList.of(new WithQuery(identifier("a"), simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("x"))), Optional.of(ImmutableList.of(identifier("t"), identifier("u")))), new WithQuery(identifier("b"), simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("y"))), Optional.empty())))), new Table(QualifiedName.of("z")), Optional.empty(), Optional.empty(), Optional.empty()));
    assertStatement("WITH RECURSIVE a AS (SELECT * FROM x) TABLE y", new Query(Optional.of(new With(true, ImmutableList.of(new WithQuery(identifier("a"), simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("x"))), Optional.empty())))), new Table(QualifiedName.of("y")), Optional.empty(), Optional.empty(), Optional.empty()));
}
Also used : CreateTable(com.facebook.presto.sql.tree.CreateTable) DropTable(com.facebook.presto.sql.tree.DropTable) Table(com.facebook.presto.sql.tree.Table) RenameTable(com.facebook.presto.sql.tree.RenameTable) Query(com.facebook.presto.sql.tree.Query) QueryUtil.simpleQuery(com.facebook.presto.sql.QueryUtil.simpleQuery) WithQuery(com.facebook.presto.sql.tree.WithQuery) WithQuery(com.facebook.presto.sql.tree.WithQuery) AllColumns(com.facebook.presto.sql.tree.AllColumns) With(com.facebook.presto.sql.tree.With) Test(org.testng.annotations.Test)

Example 8 with Query

use of com.facebook.presto.sql.tree.Query in project presto by prestodb.

the class TestSqlParser method testCreateMaterializedView.

@Test
public void testCreateMaterializedView() {
    Query query = simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("t")));
    assertStatement("CREATE MATERIALIZED VIEW mv" + " AS SELECT * FROM t", new CreateMaterializedView(Optional.empty(), QualifiedName.of("mv"), query, false, ImmutableList.of(), Optional.empty()));
    assertStatement("CREATE MATERIALIZED VIEW mv COMMENT 'A simple materialized view'" + " AS SELECT * FROM t", new CreateMaterializedView(Optional.empty(), QualifiedName.of("mv"), query, false, ImmutableList.of(), Optional.of("A simple materialized view")));
    assertStatement("CREATE MATERIALIZED VIEW IF NOT EXISTS mv COMMENT 'A simple materialized view'" + " AS SELECT * FROM t", new CreateMaterializedView(Optional.empty(), QualifiedName.of("mv"), query, true, ImmutableList.of(), Optional.of("A simple materialized view")));
    List<Property> properties = ImmutableList.of(new Property(new Identifier("partitioned_by"), new ArrayConstructor(ImmutableList.of(new StringLiteral("ds")))));
    assertStatement("CREATE MATERIALIZED VIEW IF NOT EXISTS mv COMMENT 'A simple materialized view'" + " WITH (partitioned_by = ARRAY ['ds'])" + " AS SELECT * FROM t", new CreateMaterializedView(Optional.empty(), QualifiedName.of("mv"), query, true, properties, Optional.of("A simple materialized view")));
    List<Property> properties1 = ImmutableList.of(new Property(new Identifier("partitioned_by"), new ArrayConstructor(ImmutableList.of(new StringLiteral("ds")))), new Property(new Identifier("retention_days"), new LongLiteral("90")));
    assertStatement("CREATE MATERIALIZED VIEW IF NOT EXISTS mv COMMENT 'A simple materialized view'" + " WITH (partitioned_by = ARRAY ['ds'], retention_days = 90)" + " AS SELECT * FROM t", new CreateMaterializedView(Optional.empty(), QualifiedName.of("mv"), query, true, properties1, Optional.of("A simple materialized view")));
}
Also used : CreateMaterializedView(com.facebook.presto.sql.tree.CreateMaterializedView) Identifier(com.facebook.presto.sql.tree.Identifier) QueryUtil.quotedIdentifier(com.facebook.presto.sql.QueryUtil.quotedIdentifier) Query(com.facebook.presto.sql.tree.Query) QueryUtil.simpleQuery(com.facebook.presto.sql.QueryUtil.simpleQuery) WithQuery(com.facebook.presto.sql.tree.WithQuery) StringLiteral(com.facebook.presto.sql.tree.StringLiteral) LongLiteral(com.facebook.presto.sql.tree.LongLiteral) ArrayConstructor(com.facebook.presto.sql.tree.ArrayConstructor) AllColumns(com.facebook.presto.sql.tree.AllColumns) Property(com.facebook.presto.sql.tree.Property) Test(org.testng.annotations.Test)

Example 9 with Query

use of com.facebook.presto.sql.tree.Query in project presto by prestodb.

the class DeterminismAnalyzer method analyze.

private DeterminismAnalysis analyze(QueryObjectBundle control, ChecksumResult controlChecksum, DeterminismAnalysisDetails.Builder determinismAnalysisDetails) {
    // Handle mutable catalogs
    if (isNonDeterministicCatalogReferenced(control.getQuery())) {
        return NON_DETERMINISTIC_CATALOG;
    }
    // Handle limit query
    LimitQueryDeterminismAnalysis limitQueryAnalysis = new LimitQueryDeterminismAnalyzer(prestoAction, handleLimitQuery, control.getQuery(), controlChecksum.getRowCount(), determinismAnalysisDetails).analyze();
    switch(limitQueryAnalysis) {
        case NOT_RUN:
        case FAILED_QUERY_FAILURE:
        case DETERMINISTIC:
            // try the next analysis
            break;
        case NON_DETERMINISTIC:
            return NON_DETERMINISTIC_LIMIT_CLAUSE;
        case FAILED_DATA_CHANGED:
            return ANALYSIS_FAILED_DATA_CHANGED;
        default:
            throw new IllegalArgumentException(format("Invalid limitQueryAnalysis: %s", limitQueryAnalysis));
    }
    // Rerun control query multiple times
    List<Column> columns = getColumns(prestoAction, typeManager, control.getObjectName());
    Map<QueryBundle, DeterminismAnalysisRun.Builder> queryRuns = new HashMap<>();
    try {
        for (int i = 0; i < maxAnalysisRuns; i++) {
            QueryObjectBundle queryBundle = queryRewriter.rewriteQuery(sourceQuery.getQuery(CONTROL), CONTROL);
            DeterminismAnalysisRun.Builder run = determinismAnalysisDetails.addRun().setTableName(queryBundle.getObjectName().toString());
            queryRuns.put(queryBundle, run);
            // Rerun setup and main query
            queryBundle.getSetupQueries().forEach(query -> runAndConsume(() -> prestoAction.execute(query, DETERMINISM_ANALYSIS_SETUP), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::addSetupQueryId)));
            runAndConsume(() -> prestoAction.execute(queryBundle.getQuery(), DETERMINISM_ANALYSIS_MAIN), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setQueryId));
            // Run checksum query
            Query checksumQuery = checksumValidator.generateChecksumQuery(queryBundle.getObjectName(), columns);
            ChecksumResult testChecksum = getOnlyElement(callAndConsume(() -> prestoAction.execute(checksumQuery, DETERMINISM_ANALYSIS_CHECKSUM, ChecksumResult::fromResultSet), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::setChecksumQueryId)).getResults());
            DeterminismAnalysis analysis = matchResultToDeterminism(match(checksumValidator, columns, columns, controlChecksum, testChecksum));
            if (analysis != DETERMINISTIC) {
                return analysis;
            }
        }
        return DETERMINISTIC;
    } catch (QueryException qe) {
        return ANALYSIS_FAILED_QUERY_FAILURE;
    } finally {
        if (runTeardown) {
            queryRuns.forEach((queryBundle, run) -> teardownSafely(prestoAction, Optional.of(queryBundle), queryStats -> queryStats.getQueryStats().map(QueryStats::getQueryId).ifPresent(run::addTeardownQueryId)));
        }
    }
}
Also used : NON_DETERMINISTIC_ROW_COUNT(com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_ROW_COUNT) Table(com.facebook.presto.sql.tree.Table) DataVerificationUtil.match(com.facebook.presto.verifier.framework.DataVerificationUtil.match) ANALYSIS_FAILED_DATA_CHANGED(com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_DATA_CHANGED) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) DETERMINISTIC(com.facebook.presto.verifier.framework.DeterminismAnalysis.DETERMINISTIC) VerifierUtil.callAndConsume(com.facebook.presto.verifier.framework.VerifierUtil.callAndConsume) NON_DETERMINISTIC_CATALOG(com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_CATALOG) NON_DETERMINISTIC_COLUMNS(com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_COLUMNS) DataVerificationUtil.getColumns(com.facebook.presto.verifier.framework.DataVerificationUtil.getColumns) TypeManager(com.facebook.presto.common.type.TypeManager) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) QueryStats(com.facebook.presto.jdbc.QueryStats) ANALYSIS_FAILED_INCONSISTENT_SCHEMA(com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_INCONSISTENT_SCHEMA) QueryRewriter(com.facebook.presto.verifier.rewrite.QueryRewriter) DETERMINISM_ANALYSIS_SETUP(com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS_SETUP) DeterminismAnalysisRun(com.facebook.presto.verifier.event.DeterminismAnalysisRun) ANALYSIS_FAILED_QUERY_FAILURE(com.facebook.presto.verifier.framework.DeterminismAnalysis.ANALYSIS_FAILED_QUERY_FAILURE) DETERMINISM_ANALYSIS_CHECKSUM(com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS_CHECKSUM) ImmutableSet(com.google.common.collect.ImmutableSet) DataVerificationUtil.teardownSafely(com.facebook.presto.verifier.framework.DataVerificationUtil.teardownSafely) Query(com.facebook.presto.sql.tree.Query) Node(com.facebook.presto.sql.tree.Node) AstVisitor(com.facebook.presto.sql.tree.AstVisitor) Set(java.util.Set) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) CONTROL(com.facebook.presto.verifier.framework.ClusterType.CONTROL) String.format(java.lang.String.format) ChecksumResult(com.facebook.presto.verifier.checksum.ChecksumResult) DeterminismAnalysisDetails(com.facebook.presto.verifier.event.DeterminismAnalysisDetails) List(java.util.List) VerifierUtil.runAndConsume(com.facebook.presto.verifier.framework.VerifierUtil.runAndConsume) ChecksumValidator(com.facebook.presto.verifier.checksum.ChecksumValidator) PrestoAction(com.facebook.presto.verifier.prestoaction.PrestoAction) Optional(java.util.Optional) VisibleForTesting(com.google.common.annotations.VisibleForTesting) NON_DETERMINISTIC_LIMIT_CLAUSE(com.facebook.presto.verifier.framework.DeterminismAnalysis.NON_DETERMINISTIC_LIMIT_CLAUSE) DETERMINISM_ANALYSIS_MAIN(com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS_MAIN) Statement(com.facebook.presto.sql.tree.Statement) ChecksumResult(com.facebook.presto.verifier.checksum.ChecksumResult) Query(com.facebook.presto.sql.tree.Query) HashMap(java.util.HashMap) QueryStats(com.facebook.presto.jdbc.QueryStats) DeterminismAnalysisRun(com.facebook.presto.verifier.event.DeterminismAnalysisRun)

Example 10 with Query

use of com.facebook.presto.sql.tree.Query in project presto by prestodb.

the class LimitQueryDeterminismAnalyzer method analyzeLimitNoOrderBy.

private LimitQueryDeterminismAnalysis analyzeLimitNoOrderBy(Query newLimitQuery, long limit) {
    Query rowCountQuery = simpleQuery(new Select(false, ImmutableList.of(new SingleColumn(new FunctionCall(QualifiedName.of("count"), ImmutableList.of(new LongLiteral("1")))))), new TableSubquery(newLimitQuery));
    QueryResult<Long> result = callAndConsume(() -> prestoAction.execute(rowCountQuery, DETERMINISM_ANALYSIS_MAIN, resultSet -> Optional.of(resultSet.getLong(1))), stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(determinismAnalysisDetails::setLimitQueryAnalysisQueryId));
    long rowCountHigherLimit = getOnlyElement(result.getResults());
    if (rowCountHigherLimit == rowCount) {
        return DETERMINISTIC;
    }
    if (rowCountHigherLimit > rowCount) {
        return NON_DETERMINISTIC;
    }
    return FAILED_DATA_CHANGED;
}
Also used : QualifiedName(com.facebook.presto.sql.tree.QualifiedName) NOT_RUN(com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NOT_RUN) SingleColumn(com.facebook.presto.sql.tree.SingleColumn) ArrayList(java.util.ArrayList) VerifierUtil.callAndConsume(com.facebook.presto.verifier.framework.VerifierUtil.callAndConsume) Identifier(com.facebook.presto.sql.tree.Identifier) SQLException(java.sql.SQLException) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ImmutableList(com.google.common.collect.ImmutableList) ResultSet(java.sql.ResultSet) SelectItem(com.facebook.presto.sql.tree.SelectItem) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) QueryStats(com.facebook.presto.jdbc.QueryStats) TableSubquery(com.facebook.presto.sql.tree.TableSubquery) DETERMINISTIC(com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.DETERMINISTIC) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) Math.toIntExact(java.lang.Math.toIntExact) NON_DETERMINISTIC(com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.NON_DETERMINISTIC) ENGLISH(java.util.Locale.ENGLISH) FunctionCall(com.facebook.presto.sql.tree.FunctionCall) CreateTableAsSelect(com.facebook.presto.sql.tree.CreateTableAsSelect) OrderBy(com.facebook.presto.sql.tree.OrderBy) Query(com.facebook.presto.sql.tree.Query) QuerySpecification(com.facebook.presto.sql.tree.QuerySpecification) With(com.facebook.presto.sql.tree.With) Set(java.util.Set) VerifierUtil.getColumnIndices(com.facebook.presto.verifier.framework.VerifierUtil.getColumnIndices) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) ResultSetConverter(com.facebook.presto.verifier.prestoaction.PrestoAction.ResultSetConverter) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) Insert(com.facebook.presto.sql.tree.Insert) Objects(java.util.Objects) DeterminismAnalysisDetails(com.facebook.presto.verifier.event.DeterminismAnalysisDetails) List(java.util.List) Expression(com.facebook.presto.sql.tree.Expression) FAILED_DATA_CHANGED(com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.FAILED_DATA_CHANGED) FAILED_QUERY_FAILURE(com.facebook.presto.verifier.framework.LimitQueryDeterminismAnalysis.FAILED_QUERY_FAILURE) PrestoAction(com.facebook.presto.verifier.prestoaction.PrestoAction) LongLiteral(com.facebook.presto.sql.tree.LongLiteral) Optional(java.util.Optional) VerifierUtil.delimitedIdentifier(com.facebook.presto.verifier.framework.VerifierUtil.delimitedIdentifier) Select(com.facebook.presto.sql.tree.Select) Long.parseLong(java.lang.Long.parseLong) DETERMINISM_ANALYSIS_MAIN(com.facebook.presto.verifier.framework.QueryStage.DETERMINISM_ANALYSIS_MAIN) Statement(com.facebook.presto.sql.tree.Statement) QueryUtil.simpleQuery(com.facebook.presto.sql.QueryUtil.simpleQuery) Query(com.facebook.presto.sql.tree.Query) QueryUtil.simpleQuery(com.facebook.presto.sql.QueryUtil.simpleQuery) LongLiteral(com.facebook.presto.sql.tree.LongLiteral) QueryStats(com.facebook.presto.jdbc.QueryStats) CreateTableAsSelect(com.facebook.presto.sql.tree.CreateTableAsSelect) Select(com.facebook.presto.sql.tree.Select) Long.parseLong(java.lang.Long.parseLong) SingleColumn(com.facebook.presto.sql.tree.SingleColumn) FunctionCall(com.facebook.presto.sql.tree.FunctionCall) TableSubquery(com.facebook.presto.sql.tree.TableSubquery)

Aggregations

Query (com.facebook.presto.sql.tree.Query)40 QueryUtil.simpleQuery (com.facebook.presto.sql.QueryUtil.simpleQuery)21 WithQuery (com.facebook.presto.sql.tree.WithQuery)18 Test (org.testng.annotations.Test)18 QuerySpecification (com.facebook.presto.sql.tree.QuerySpecification)17 AllColumns (com.facebook.presto.sql.tree.AllColumns)16 Identifier (com.facebook.presto.sql.tree.Identifier)13 LongLiteral (com.facebook.presto.sql.tree.LongLiteral)12 Table (com.facebook.presto.sql.tree.Table)12 CreateTableAsSelect (com.facebook.presto.sql.tree.CreateTableAsSelect)9 FunctionCall (com.facebook.presto.sql.tree.FunctionCall)8 OrderBy (com.facebook.presto.sql.tree.OrderBy)8 StringLiteral (com.facebook.presto.sql.tree.StringLiteral)8 QueryUtil.quotedIdentifier (com.facebook.presto.sql.QueryUtil.quotedIdentifier)7 CreateTable (com.facebook.presto.sql.tree.CreateTable)7 DropTable (com.facebook.presto.sql.tree.DropTable)7 QualifiedName (com.facebook.presto.sql.tree.QualifiedName)7 RenameTable (com.facebook.presto.sql.tree.RenameTable)6 QualifiedObjectName (com.facebook.presto.common.QualifiedObjectName)5 Expression (com.facebook.presto.sql.tree.Expression)5