Examples with DistributedQueryRunner com.facebook.presto.tests.DistributedQueryRunner used on opensource projects

Example 41 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class ThriftQueryRunner method createThriftQueryRunner.

public static QueryRunner createThriftQueryRunner(int thriftServers, int nodeCount, boolean enableIndexJoin, Map<String, String> properties) throws Exception {
    List<DriftServer> servers = null;
    DistributedQueryRunner runner = null;
    try {
        servers = startThriftServers(thriftServers, enableIndexJoin);
        runner = createThriftQueryRunnerInternal(servers, nodeCount, properties);
        return new ThriftQueryRunnerWithServers(runner, servers);
    } catch (Throwable t) {
        closeQuietly(runner);
        // runner might be null, so closing servers explicitly
        if (servers != null) {
            for (DriftServer server : servers) {
                server.shutdown();
            }
        }
        throw t;
    }
}

Example 42 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class BigQueryQueryRunner method createQueryRunner.

public static DistributedQueryRunner createQueryRunner(Map<String, String> extraProperties) throws Exception {
    DistributedQueryRunner queryRunner = null;
    try {
        queryRunner = DistributedQueryRunner.builder(createSession()).setExtraProperties(extraProperties).build();
        queryRunner.installPlugin(new TpchPlugin());
        queryRunner.createCatalog("tpch", "tpch");
        queryRunner.installPlugin(new BigQueryPlugin());
        queryRunner.createCatalog("bigquery", "bigquery");
        return queryRunner;
    } catch (Throwable e) {
        closeAllSuppress(e, queryRunner);
        throw e;
    }
}

Example 43 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestHiveLogicalPlanner method testBaseToViewConversionWithDerivedFields.

@Test
public void testBaseToViewConversionWithDerivedFields() {
    Session queryOptimizationWithMaterializedView = Session.builder(getSession()).setSystemProperty(QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED, "true").build();
    QueryRunner queryRunner = getQueryRunner();
    String table = "lineitem_partitioned_derived_fields";
    String view = "lineitem_partitioned_view_derived_fields";
    try {
        queryRunner.execute(format("CREATE TABLE %s WITH (partitioned_by = ARRAY['ds', 'shipmode']) AS " + "SELECT discount, extendedprice, '2020-01-01' as ds, shipmode FROM lineitem WHERE orderkey < 1000 " + "UNION ALL " + "SELECT discount, extendedprice, '2020-01-02' as ds, shipmode FROM lineitem WHERE orderkey > 1000", table));
        assertUpdate(format("CREATE MATERIALIZED VIEW %s WITH (partitioned_by = ARRAY['mvds', 'shipmode']) AS " + "SELECT SUM(discount * extendedprice) as _discount_multi_extendedprice_ , MAX(discount*extendedprice) as _max_discount_multi_extendedprice_ , " + "ds as mvds, shipmode FROM %s group by ds, shipmode", view, table));
        assertTrue(getQueryRunner().tableExists(getSession(), view));
        setReferencedMaterializedViews((DistributedQueryRunner) queryRunner, table, ImmutableList.of(view));
        assertUpdate(format("REFRESH MATERIALIZED VIEW %s where mvds='2020-01-01'", view), 7);
        String baseQuery = format("SELECT sum(discount * extendedprice) as _discount_multi_extendedprice_ , MAX(discount*extendedprice) as _max_discount_multi_extendedprice_ , " + "ds, shipmode as method from %s group by ds, shipmode ORDER BY ds, shipmode", table);
        String viewQuery = format("SELECT _discount_multi_extendedprice_ , _max_discount_multi_extendedprice_ , " + "mvds, shipmode as method from %s ORDER BY mvds, shipmode", view);
        MaterializedResult optimizedQueryResult = computeActual(queryOptimizationWithMaterializedView, baseQuery);
        MaterializedResult baseQueryResult = computeActual(baseQuery);
        assertEquals(optimizedQueryResult, baseQueryResult);
        assertPlan(getSession(), viewQuery, anyTree(anyTree(PlanMatchPattern.constrainedTableScan(table, ImmutableMap.of("shipmode", multipleValues(createVarcharType(10), utf8Slices("AIR", "FOB", "MAIL", "RAIL", "REG AIR", "SHIP", "TRUCK")), "ds", singleValue(createVarcharType(10), utf8Slice("2020-01-02"))))), PlanMatchPattern.constrainedTableScan(view, ImmutableMap.of())));
        assertPlan(queryOptimizationWithMaterializedView, baseQuery, anyTree(anyTree(PlanMatchPattern.constrainedTableScan(table, ImmutableMap.of("shipmode", multipleValues(createVarcharType(10), utf8Slices("AIR", "FOB", "MAIL", "RAIL", "REG AIR", "SHIP", "TRUCK")), "ds", singleValue(createVarcharType(10), utf8Slice("2020-01-02"))))), anyTree(PlanMatchPattern.constrainedTableScan(view, ImmutableMap.of()))));
    } finally {
        queryRunner.execute("DROP MATERIALIZED VIEW IF EXISTS " + view);
        queryRunner.execute("DROP TABLE IF EXISTS " + table);
    }
}

Example 44 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestHiveLogicalPlanner method testBaseToViewConversionWithGroupBy.

@Test
public void testBaseToViewConversionWithGroupBy() {
    Session queryOptimizationWithMaterializedView = Session.builder(getSession()).setSystemProperty(QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED, "true").build();
    QueryRunner queryRunner = getQueryRunner();
    String table = "lineitem_partitioned_derived_fields";
    String view = "lineitem_partitioned_view_derived_fields";
    try {
        queryRunner.execute(format("CREATE TABLE %s WITH (partitioned_by = ARRAY['ds', 'shipmode']) AS " + "SELECT discount, extendedprice, '2020-01-01' as ds, shipmode FROM lineitem WHERE orderkey < 1000 " + "UNION ALL " + "SELECT discount, extendedprice, '2020-01-02' as ds, shipmode FROM lineitem WHERE orderkey > 1000", table));
        assertUpdate(format("CREATE MATERIALIZED VIEW %s WITH (partitioned_by = ARRAY['ds', 'shipmode']) AS " + "SELECT SUM(discount * extendedprice) as _discount_multi_extendedprice_ , MAX(discount*extendedprice) as _max_discount_multi_extendedprice_ , " + "ds, shipmode FROM %s group by ds, shipmode", view, table));
        assertTrue(getQueryRunner().tableExists(getSession(), view));
        setReferencedMaterializedViews((DistributedQueryRunner) queryRunner, table, ImmutableList.of(view));
        assertUpdate(format("REFRESH MATERIALIZED VIEW %s where ds='2020-01-01'", view), 7);
        String baseQuery = format("SELECT SUM(discount * extendedprice) as _discount_multi_extendedprice_, MAX(discount*extendedprice) as _max_discount_multi_extendedprice_ FROM %s", table);
        String viewQuery = format("SELECT SUM(_discount_multi_extendedprice_), MAX(_max_discount_multi_extendedprice_) FROM %s", view);
        MaterializedResult optimizedQueryResult = computeActual(queryOptimizationWithMaterializedView, baseQuery);
        MaterializedResult baseQueryResult = computeActual(baseQuery);
        assertEquals(optimizedQueryResult, baseQueryResult);
        PlanMatchPattern expectedPattern = anyTree(anyTree(PlanMatchPattern.constrainedTableScan(table, ImmutableMap.of("shipmode", multipleValues(createVarcharType(10), utf8Slices("AIR", "FOB", "MAIL", "RAIL", "REG AIR", "SHIP", "TRUCK")), "ds", singleValue(createVarcharType(10), utf8Slice("2020-01-02"))))), anyTree(PlanMatchPattern.constrainedTableScan(view, ImmutableMap.of())));
        assertPlan(getSession(), viewQuery, expectedPattern);
        assertPlan(queryOptimizationWithMaterializedView, baseQuery, expectedPattern);
    } finally {
        queryRunner.execute("DROP MATERIALIZED VIEW IF EXISTS " + view);
        queryRunner.execute("DROP TABLE IF EXISTS " + table);
    }
}

Example 45 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestHiveLogicalPlanner method testMetadataAggregationFoldingWithEmptyPartitionsAndMetastoreThreshold.

@Test
public void testMetadataAggregationFoldingWithEmptyPartitionsAndMetastoreThreshold() {
    QueryRunner queryRunner = getQueryRunner();
    Session optimizeMetadataQueriesWithHighThreshold = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(OPTIMIZE_METADATA_QUERIES, Boolean.toString(true)).setSystemProperty(OPTIMIZE_METADATA_QUERIES_CALL_THRESHOLD, Integer.toString(100)).build();
    Session optimizeMetadataQueriesWithLowThreshold = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(OPTIMIZE_METADATA_QUERIES, Boolean.toString(true)).setSystemProperty(OPTIMIZE_METADATA_QUERIES_CALL_THRESHOLD, Integer.toString(1)).build();
    Session optimizeMetadataQueriesIgnoreStatsWithLowThreshold = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(OPTIMIZE_METADATA_QUERIES_IGNORE_STATS, Boolean.toString(true)).setSystemProperty(OPTIMIZE_METADATA_QUERIES_CALL_THRESHOLD, Integer.toString(1)).build();
    Session shufflePartitionColumns = Session.builder(this.getQueryRunner().getDefaultSession()).setCatalogSessionProperty(HIVE_CATALOG, SHUFFLE_PARTITIONED_COLUMNS_FOR_TABLE_WRITE, Boolean.toString(true)).build();
    queryRunner.execute(shufflePartitionColumns, "CREATE TABLE test_metadata_aggregation_folding_with_empty_partitions_with_threshold WITH (partitioned_by = ARRAY['ds']) AS " + "SELECT orderkey, CAST(to_iso8601(date_add('DAY', orderkey % 2, date('2020-07-01'))) AS VARCHAR) AS ds FROM orders WHERE orderkey < 1000");
    ExtendedHiveMetastore metastore = replicateHiveMetastore((DistributedQueryRunner) queryRunner);
    MetastoreContext metastoreContext = new MetastoreContext(getSession().getUser(), getSession().getQueryId().getId(), Optional.empty(), Optional.empty(), Optional.empty(), false, HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER);
    Table table = metastore.getTable(metastoreContext, getSession().getSchema().get(), "test_metadata_aggregation_folding_with_empty_partitions_with_threshold").get();
    // Add one partition with no statistics.
    String partitionNameNoStats = "ds=2020-07-20";
    Partition partitionNoStats = createDummyPartition(table, partitionNameNoStats);
    metastore.addPartitions(metastoreContext, table.getDatabaseName(), table.getTableName(), ImmutableList.of(new PartitionWithStatistics(partitionNoStats, partitionNameNoStats, PartitionStatistics.empty())));
    // Add one partition with statistics indicating that it has no rows.
    String emptyPartitionName = "ds=2020-06-30";
    Partition emptyPartition = createDummyPartition(table, emptyPartitionName);
    metastore.addPartitions(metastoreContext, table.getDatabaseName(), table.getTableName(), ImmutableList.of(new PartitionWithStatistics(emptyPartition, emptyPartitionName, PartitionStatistics.builder().setBasicStatistics(new HiveBasicStatistics(1, 0, 0, 0)).build())));
    try {
        // Test the non-reducible code path.
        // Enable rewrite as all matching partitions have stats.
        assertPlan(optimizeMetadataQueriesWithHighThreshold, "SELECT DISTINCT ds FROM test_metadata_aggregation_folding_with_empty_partitions_with_threshold WHERE ds BETWEEN '2020-07-01' AND '2020-07-03'", anyTree(values(ImmutableList.of("ds"), ImmutableList.of(ImmutableList.of(new StringLiteral("2020-07-01")), ImmutableList.of(new StringLiteral("2020-07-02"))))));
        // All matching partitions have stats, Metastore threshold is reached, Disable rewrite
        assertPlan(optimizeMetadataQueriesWithLowThreshold, "SELECT DISTINCT ds FROM test_metadata_aggregation_folding_with_empty_partitions_with_threshold WHERE ds BETWEEN '2020-07-01' AND '2020-07-03'", anyTree(tableScanWithConstraint("test_metadata_aggregation_folding_with_empty_partitions_with_threshold", ImmutableMap.of("ds", multipleValues(VARCHAR, utf8Slices("2020-07-01", "2020-07-02"))))));
        // All matching partitions have stats, Metastore threshold is reached, but IgnoreStats will overwrite, Enable rewrite
        assertPlan(optimizeMetadataQueriesIgnoreStatsWithLowThreshold, "SELECT DISTINCT ds FROM test_metadata_aggregation_folding_with_empty_partitions_with_threshold WHERE ds BETWEEN '2020-07-01' AND '2020-07-03'", anyTree(values(ImmutableList.of("ds"), ImmutableList.of(ImmutableList.of(new StringLiteral("2020-07-01")), ImmutableList.of(new StringLiteral("2020-07-02"))))));
    } finally {
        queryRunner.execute("DROP TABLE IF EXISTS test_metadata_aggregation_folding_with_empty_partitions_with_threshold");
    }
}