Search in sources :

Example 1 with QueryObjectBundle

use of com.facebook.presto.verifier.framework.QueryObjectBundle in project presto by prestodb.

the class TooManyOpenPartitionsFailureResolver method resolveQueryFailure.

@Override
public Optional<String> resolveQueryFailure(QueryStats controlQueryStats, QueryException queryException, Optional<QueryObjectBundle> test) {
    if (!test.isPresent()) {
        return Optional.empty();
    }
    return mapMatchingPrestoException(queryException, TEST_MAIN, ImmutableSet.of(HIVE_TOO_MANY_OPEN_PARTITIONS), e -> {
        try {
            ShowCreate showCreate = new ShowCreate(TABLE, test.get().getObjectName());
            String showCreateResult = getOnlyElement(prestoAction.execute(showCreate, DESCRIBE, resultSet -> Optional.of(resultSet.getString(1))).getResults());
            CreateTable createTable = (CreateTable) sqlParser.createStatement(showCreateResult, ParsingOptions.builder().setDecimalLiteralTreatment(AS_DOUBLE).build());
            List<Property> bucketCountProperty = createTable.getProperties().stream().filter(property -> property.getName().getValue().equals(BUCKET_COUNT_PROPERTY)).collect(toImmutableList());
            if (bucketCountProperty.size() != 1) {
                return Optional.empty();
            }
            long bucketCount = ((LongLiteral) getOnlyElement(bucketCountProperty).getValue()).getValue();
            int testClusterSize = this.testClusterSizeSupplier.get();
            if (testClusterSize * maxBucketPerWriter < bucketCount) {
                return Optional.of("Not enough workers on test cluster");
            }
            return Optional.empty();
        } catch (Throwable t) {
            log.warn(t, "Exception when resolving HIVE_TOO_MANY_OPEN_PARTITIONS");
            return Optional.empty();
        }
    });
}
Also used : Logger(com.facebook.airlift.log.Logger) TEST_MAIN(com.facebook.presto.verifier.framework.QueryStage.TEST_MAIN) Suppliers.memoizeWithExpiration(com.google.common.base.Suppliers.memoizeWithExpiration) Supplier(java.util.function.Supplier) Duration(io.airlift.units.Duration) Inject(javax.inject.Inject) QueryObjectBundle(com.facebook.presto.verifier.framework.QueryObjectBundle) DESCRIBE(com.facebook.presto.verifier.framework.QueryStage.DESCRIBE) Objects.requireNonNull(java.util.Objects.requireNonNull) QueryStats(com.facebook.presto.jdbc.QueryStats) CreateTable(com.facebook.presto.sql.tree.CreateTable) ForTest(com.facebook.presto.verifier.annotation.ForTest) Property(com.facebook.presto.sql.tree.Property) ImmutableSet(com.google.common.collect.ImmutableSet) FailureResolverUtil.mapMatchingPrestoException(com.facebook.presto.verifier.resolver.FailureResolverUtil.mapMatchingPrestoException) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) HIVE_TOO_MANY_OPEN_PARTITIONS(com.facebook.presto.hive.HiveErrorCode.HIVE_TOO_MANY_OPEN_PARTITIONS) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) SqlParser(com.facebook.presto.sql.parser.SqlParser) QueryException(com.facebook.presto.verifier.framework.QueryException) List(java.util.List) AS_DOUBLE(com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) ShowCreate(com.facebook.presto.sql.tree.ShowCreate) PrestoAction(com.facebook.presto.verifier.prestoaction.PrestoAction) LongLiteral(com.facebook.presto.sql.tree.LongLiteral) Optional(java.util.Optional) ParsingOptions(com.facebook.presto.sql.parser.ParsingOptions) TABLE(com.facebook.presto.sql.tree.ShowCreate.Type.TABLE) LongLiteral(com.facebook.presto.sql.tree.LongLiteral) ShowCreate(com.facebook.presto.sql.tree.ShowCreate) CreateTable(com.facebook.presto.sql.tree.CreateTable) Property(com.facebook.presto.sql.tree.Property)

Example 2 with QueryObjectBundle

use of com.facebook.presto.verifier.framework.QueryObjectBundle in project presto by prestodb.

the class QueryRewriter method rewriteQuery.

public QueryObjectBundle rewriteQuery(@Language("SQL") String query, ClusterType clusterType) {
    checkState(prefixes.containsKey(clusterType), "Unsupported cluster type: %s", clusterType);
    Statement statement = sqlParser.createStatement(query, PARSING_OPTIONS);
    QualifiedName prefix = prefixes.get(clusterType);
    List<Property> properties = tableProperties.get(clusterType);
    if (statement instanceof CreateTableAsSelect) {
        CreateTableAsSelect createTableAsSelect = (CreateTableAsSelect) statement;
        QualifiedName temporaryTableName = generateTemporaryName(Optional.of(createTableAsSelect.getName()), prefix);
        return new QueryObjectBundle(temporaryTableName, ImmutableList.of(), new CreateTableAsSelect(temporaryTableName, createTableAsSelect.getQuery(), createTableAsSelect.isNotExists(), applyPropertyOverride(createTableAsSelect.getProperties(), properties), createTableAsSelect.isWithData(), createTableAsSelect.getColumnAliases(), createTableAsSelect.getComment()), ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType);
    }
    if (statement instanceof Insert) {
        Insert insert = (Insert) statement;
        QualifiedName originalTableName = insert.getTarget();
        QualifiedName temporaryTableName = generateTemporaryName(Optional.of(originalTableName), prefix);
        return new QueryObjectBundle(temporaryTableName, ImmutableList.of(new CreateTable(temporaryTableName, ImmutableList.of(new LikeClause(originalTableName, Optional.of(INCLUDING))), false, properties, Optional.empty())), new Insert(temporaryTableName, insert.getColumns(), insert.getQuery()), ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType);
    }
    if (statement instanceof Query) {
        QualifiedName temporaryTableName = generateTemporaryName(Optional.empty(), prefix);
        ResultSetMetaData metadata = getResultMetadata((Query) statement);
        List<Identifier> columnAliases = generateStorageColumnAliases(metadata);
        Query rewrite = rewriteNonStorableColumns((Query) statement, metadata);
        return new QueryObjectBundle(temporaryTableName, ImmutableList.of(), new CreateTableAsSelect(temporaryTableName, rewrite, false, properties, true, Optional.of(columnAliases), Optional.empty()), ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType);
    }
    if (statement instanceof CreateView) {
        CreateView createView = (CreateView) statement;
        QualifiedName temporaryViewName = generateTemporaryName(Optional.empty(), prefix);
        ImmutableList.Builder<Statement> setupQueries = ImmutableList.builder();
        // Otherwise, do not pre-create temporary view.
        try {
            String createExistingViewQuery = getOnlyElement(prestoAction.execute(new ShowCreate(VIEW, createView.getName()), REWRITE, SHOW_CREATE_VIEW_CONVERTER).getResults());
            CreateView createExistingView = (CreateView) sqlParser.createStatement(createExistingViewQuery, PARSING_OPTIONS);
            setupQueries.add(new CreateView(temporaryViewName, createExistingView.getQuery(), false, createExistingView.getSecurity()));
        } catch (QueryException e) {
        // no-op
        }
        return new QueryObjectBundle(temporaryViewName, setupQueries.build(), new CreateView(temporaryViewName, createView.getQuery(), createView.isReplace(), createView.getSecurity()), ImmutableList.of(new DropView(temporaryViewName, true)), clusterType);
    }
    if (statement instanceof CreateTable) {
        CreateTable createTable = (CreateTable) statement;
        QualifiedName temporaryTableName = generateTemporaryName(Optional.empty(), prefix);
        return new QueryObjectBundle(temporaryTableName, ImmutableList.of(), new CreateTable(temporaryTableName, createTable.getElements(), createTable.isNotExists(), applyPropertyOverride(createTable.getProperties(), properties), createTable.getComment()), ImmutableList.of(new DropTable(temporaryTableName, true)), clusterType);
    }
    throw new IllegalStateException(format("Unsupported query type: %s", statement.getClass()));
}
Also used : LikeClause(com.facebook.presto.sql.tree.LikeClause) Query(com.facebook.presto.sql.tree.Query) Statement(com.facebook.presto.sql.tree.Statement) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) QualifiedName(com.facebook.presto.sql.tree.QualifiedName) DropView(com.facebook.presto.sql.tree.DropView) CreateTable(com.facebook.presto.sql.tree.CreateTable) DropTable(com.facebook.presto.sql.tree.DropTable) Insert(com.facebook.presto.sql.tree.Insert) CreateView(com.facebook.presto.sql.tree.CreateView) ResultSetMetaData(java.sql.ResultSetMetaData) QueryException(com.facebook.presto.verifier.framework.QueryException) Identifier(com.facebook.presto.sql.tree.Identifier) ShowCreate(com.facebook.presto.sql.tree.ShowCreate) CreateTableAsSelect(com.facebook.presto.sql.tree.CreateTableAsSelect) QueryObjectBundle(com.facebook.presto.verifier.framework.QueryObjectBundle) Property(com.facebook.presto.sql.tree.Property)

Example 3 with QueryObjectBundle

use of com.facebook.presto.verifier.framework.QueryObjectBundle in project presto by prestodb.

the class TestQueryRewriter method assertTableName.

private static void assertTableName(QueryRewriter queryRewriter, @Language("SQL") String query, String expectedPrefix) {
    QueryObjectBundle bundle = queryRewriter.rewriteQuery(query, CONTROL);
    assertTrue(bundle.getObjectName().toString().startsWith(expectedPrefix));
}
Also used : QueryObjectBundle(com.facebook.presto.verifier.framework.QueryObjectBundle)

Example 4 with QueryObjectBundle

use of com.facebook.presto.verifier.framework.QueryObjectBundle in project presto by prestodb.

the class TestQueryRewriter method assertShadowed.

private void assertShadowed(QueryRewriter queryRewriter, @Language("SQL") String query, String prefix, List<String> expectedSetupTemplates, @Language("SQL") String expectedTemplates, List<String> expectedTeardownTemplates) {
    for (ClusterType cluster : ClusterType.values()) {
        QueryObjectBundle bundle = queryRewriter.rewriteQuery(query, cluster);
        String tableName = bundle.getObjectName().toString();
        assertTrue(tableName.startsWith(prefix + "_"));
        assertStatements(bundle.getSetupQueries(), templateToStatements(expectedSetupTemplates, tableName));
        assertStatements(ImmutableList.of(bundle.getQuery()), templateToStatements(ImmutableList.of(expectedTemplates), tableName));
        assertStatements(bundle.getTeardownQueries(), templateToStatements(expectedTeardownTemplates, tableName));
    }
}
Also used : QueryObjectBundle(com.facebook.presto.verifier.framework.QueryObjectBundle) ClusterType(com.facebook.presto.verifier.framework.ClusterType)

Aggregations

QueryObjectBundle (com.facebook.presto.verifier.framework.QueryObjectBundle)4 CreateTable (com.facebook.presto.sql.tree.CreateTable)2 Property (com.facebook.presto.sql.tree.Property)2 ShowCreate (com.facebook.presto.sql.tree.ShowCreate)2 QueryException (com.facebook.presto.verifier.framework.QueryException)2 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)2 Logger (com.facebook.airlift.log.Logger)1 HIVE_TOO_MANY_OPEN_PARTITIONS (com.facebook.presto.hive.HiveErrorCode.HIVE_TOO_MANY_OPEN_PARTITIONS)1 BUCKET_COUNT_PROPERTY (com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY)1 QueryStats (com.facebook.presto.jdbc.QueryStats)1 ParsingOptions (com.facebook.presto.sql.parser.ParsingOptions)1 AS_DOUBLE (com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DOUBLE)1 SqlParser (com.facebook.presto.sql.parser.SqlParser)1 CreateTableAsSelect (com.facebook.presto.sql.tree.CreateTableAsSelect)1 CreateView (com.facebook.presto.sql.tree.CreateView)1 DropTable (com.facebook.presto.sql.tree.DropTable)1 DropView (com.facebook.presto.sql.tree.DropView)1 Identifier (com.facebook.presto.sql.tree.Identifier)1 Insert (com.facebook.presto.sql.tree.Insert)1 LikeClause (com.facebook.presto.sql.tree.LikeClause)1