Examples with DistributedQueryRunner com.facebook.presto.tests.DistributedQueryRunner used on opensource projects

Example 91 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestHiveLogicalPlanner method testMetadataAggregationFoldingWithEmptyPartitions.

@Test
public void testMetadataAggregationFoldingWithEmptyPartitions() {
    QueryRunner queryRunner = getQueryRunner();
    Session optimizeMetadataQueries = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(OPTIMIZE_METADATA_QUERIES, Boolean.toString(true)).build();
    Session optimizeMetadataQueriesIgnoreStats = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(OPTIMIZE_METADATA_QUERIES_IGNORE_STATS, Boolean.toString(true)).build();
    Session shufflePartitionColumns = Session.builder(this.getQueryRunner().getDefaultSession()).setCatalogSessionProperty(HIVE_CATALOG, SHUFFLE_PARTITIONED_COLUMNS_FOR_TABLE_WRITE, Boolean.toString(true)).build();
    queryRunner.execute(shufflePartitionColumns, "CREATE TABLE test_metadata_aggregation_folding_with_empty_partitions WITH (partitioned_by = ARRAY['ds']) AS " + "SELECT orderkey, CAST(to_iso8601(date_add('DAY', orderkey % 2, date('2020-07-01'))) AS VARCHAR) AS ds FROM orders WHERE orderkey < 1000");
    ExtendedHiveMetastore metastore = replicateHiveMetastore((DistributedQueryRunner) queryRunner);
    MetastoreContext metastoreContext = new MetastoreContext(getSession().getUser(), getSession().getQueryId().getId(), Optional.empty(), Optional.empty(), Optional.empty(), false, HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER);
    Table table = metastore.getTable(metastoreContext, getSession().getSchema().get(), "test_metadata_aggregation_folding_with_empty_partitions").get();
    // Add one partition with no statistics.
    String partitionNameNoStats = "ds=2020-07-20";
    Partition partitionNoStats = createDummyPartition(table, partitionNameNoStats);
    metastore.addPartitions(metastoreContext, table.getDatabaseName(), table.getTableName(), ImmutableList.of(new PartitionWithStatistics(partitionNoStats, partitionNameNoStats, PartitionStatistics.empty())));
    // Add one partition with statistics indicating that it has no rows.
    String emptyPartitionName = "ds=2020-06-30";
    Partition emptyPartition = createDummyPartition(table, emptyPartitionName);
    metastore.addPartitions(metastoreContext, table.getDatabaseName(), table.getTableName(), ImmutableList.of(new PartitionWithStatistics(emptyPartition, emptyPartitionName, PartitionStatistics.builder().setBasicStatistics(new HiveBasicStatistics(1, 0, 0, 0)).build())));
    try {
        // Max ds doesn't have stats. Disable rewrite.
        assertPlan(optimizeMetadataQueries, "SELECT * FROM test_metadata_aggregation_folding_with_empty_partitions WHERE ds = (SELECT max(ds) from test_metadata_aggregation_folding_with_empty_partitions)", anyTree(anyTree(PlanMatchPattern.tableScan("test_metadata_aggregation_folding_with_empty_partitions")), anyTree(PlanMatchPattern.tableScan("test_metadata_aggregation_folding_with_empty_partitions"))));
        // Ignore metastore stats. Enable rewrite.
        assertPlan(optimizeMetadataQueriesIgnoreStats, "SELECT * FROM test_metadata_aggregation_folding_with_empty_partitions WHERE ds = (SELECT max(ds) from test_metadata_aggregation_folding_with_empty_partitions)", anyTree(join(INNER, ImmutableList.of(), tableScan("test_metadata_aggregation_folding_with_empty_partitions", getSingleValueColumnDomain("ds", "2020-07-20"), TRUE_CONSTANT, ImmutableSet.of("ds")), anyTree(any()))));
        // Max ds matching the filter has stats. Enable rewrite.
        assertPlan(optimizeMetadataQueries, "SELECT * FROM test_metadata_aggregation_folding_with_empty_partitions WHERE ds = (SELECT max(ds) from test_metadata_aggregation_folding_with_empty_partitions WHERE ds <= '2020-07-02')", anyTree(join(INNER, ImmutableList.of(), tableScan("test_metadata_aggregation_folding_with_empty_partitions", getSingleValueColumnDomain("ds", "2020-07-02"), TRUE_CONSTANT, ImmutableSet.of("ds")), anyTree(any()))));
        // Min ds partition stats indicates that it is an empty partition. Disable rewrite.
        assertPlan(optimizeMetadataQueries, "SELECT * FROM test_metadata_aggregation_folding_with_empty_partitions WHERE ds = (SELECT min(ds) from test_metadata_aggregation_folding_with_empty_partitions)", anyTree(anyTree(PlanMatchPattern.tableScan("test_metadata_aggregation_folding_with_empty_partitions")), anyTree(PlanMatchPattern.tableScan("test_metadata_aggregation_folding_with_empty_partitions"))));
        // Min ds partition matching the filter has non-empty stats. Enable rewrite.
        assertPlan(optimizeMetadataQueries, "SELECT * FROM test_metadata_aggregation_folding_with_empty_partitions WHERE ds = (SELECT min(ds) from test_metadata_aggregation_folding_with_empty_partitions WHERE ds >= '2020-07-01')", anyTree(join(INNER, ImmutableList.of(), tableScan("test_metadata_aggregation_folding_with_empty_partitions", getSingleValueColumnDomain("ds", "2020-07-01"), TRUE_CONSTANT, ImmutableSet.of("ds")), anyTree(any()))));
        // Test the non-reducible code path.
        // Disable rewrite as there are partitions with empty stats.
        assertPlan(optimizeMetadataQueries, "SELECT DISTINCT ds FROM test_metadata_aggregation_folding_with_empty_partitions", anyTree(tableScanWithConstraint("test_metadata_aggregation_folding_with_empty_partitions", ImmutableMap.of("ds", multipleValues(VARCHAR, utf8Slices("2020-06-30", "2020-07-01", "2020-07-02", "2020-07-20"))))));
        // Enable rewrite as all matching partitions have stats.
        assertPlan(optimizeMetadataQueries, "SELECT DISTINCT ds FROM test_metadata_aggregation_folding_with_empty_partitions WHERE ds BETWEEN '2020-07-01' AND '2020-07-03'", anyTree(values(ImmutableList.of("ds"), ImmutableList.of(ImmutableList.of(new StringLiteral("2020-07-01")), ImmutableList.of(new StringLiteral("2020-07-02"))))));
        // One of two resulting partitions doesn't have stats. Disable rewrite.
        assertPlan(optimizeMetadataQueries, "SELECT MIN(ds), MAX(ds) FROM test_metadata_aggregation_folding_with_empty_partitions WHERE ds BETWEEN '2020-06-30' AND '2020-07-03'", anyTree(tableScanWithConstraint("test_metadata_aggregation_folding_with_empty_partitions", ImmutableMap.of("ds", multipleValues(VARCHAR, utf8Slices("2020-06-30", "2020-07-01", "2020-07-02"))))));
        // Ignore metadata stats. Always enable rewrite.
        assertPlan(optimizeMetadataQueriesIgnoreStats, "SELECT MIN(ds), MAX(ds) FROM test_metadata_aggregation_folding_with_empty_partitions WHERE ds BETWEEN '2020-06-30' AND '2020-07-03'", anyTree(project(ImmutableMap.of("min", expression("'2020-06-30'"), "max", expression("'2020-07-02'")), anyTree(values()))));
        // Both resulting partitions have stats. Enable rewrite.
        assertPlan(optimizeMetadataQueries, "SELECT MIN(ds), MAX(ds) FROM test_metadata_aggregation_folding_with_empty_partitions WHERE ds BETWEEN '2020-07-01' AND '2020-07-03'", anyTree(project(ImmutableMap.of("min", expression("'2020-07-01'"), "max", expression("'2020-07-02'")), anyTree(values()))));
    } finally {
        queryRunner.execute("DROP TABLE IF EXISTS test_metadata_aggregation_folding_with_empty_partitions");
    }
}

Example 92 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestHiveLogicalPlanner method testMetadataAggregationFoldingWithTwoPartitionColumns.

@Test
public void testMetadataAggregationFoldingWithTwoPartitionColumns() {
    QueryRunner queryRunner = getQueryRunner();
    Session optimizeMetadataQueries = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(OPTIMIZE_METADATA_QUERIES, Boolean.toString(true)).build();
    Session shufflePartitionColumns = Session.builder(this.getQueryRunner().getDefaultSession()).setCatalogSessionProperty(HIVE_CATALOG, SHUFFLE_PARTITIONED_COLUMNS_FOR_TABLE_WRITE, Boolean.toString(true)).build();
    // Create a table with partitions: ds=2020-07-01/status=A, ds=2020-07-01/status=B, ds=2020-07-02/status=A, ds=2020-07-02/status=B
    queryRunner.execute(shufflePartitionColumns, "CREATE TABLE test_metadata_aggregation_folding_with_two_partitions_columns WITH (partitioned_by = ARRAY['ds', 'status']) AS " + "SELECT orderkey, CAST(to_iso8601(date_add('DAY', orderkey % 2, date('2020-07-01'))) AS VARCHAR) AS ds, IF(orderkey % 2 = 1, 'A', 'B') status " + "FROM orders WHERE orderkey < 1000");
    ExtendedHiveMetastore metastore = replicateHiveMetastore((DistributedQueryRunner) queryRunner);
    MetastoreContext metastoreContext = new MetastoreContext(getSession().getUser(), getSession().getQueryId().getId(), Optional.empty(), Optional.empty(), Optional.empty(), false, HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER);
    Table table = metastore.getTable(metastoreContext, getSession().getSchema().get(), "test_metadata_aggregation_folding_with_two_partitions_columns").get();
    // Add one partition with no statistics.
    String partitionNameNoStats = "ds=2020-07-03/status=C";
    Partition partitionNoStats = createDummyPartition(table, partitionNameNoStats);
    metastore.addPartitions(metastoreContext, table.getDatabaseName(), table.getTableName(), ImmutableList.of(new PartitionWithStatistics(partitionNoStats, partitionNameNoStats, PartitionStatistics.empty())));
    try {
        // All matching partitions have stats. Enable rewrite.
        assertPlan(optimizeMetadataQueries, "SELECT MIN(ds), MAX(ds) FROM test_metadata_aggregation_folding_with_two_partitions_columns WHERE ds BETWEEN '2020-07-01' AND '2020-07-02'", anyTree(project(ImmutableMap.of("min", expression("'2020-07-01'"), "max", expression("'2020-07-02'")), anyTree(values()))));
        // All matching partitions have stats. Enable rewrite.
        assertPlan(optimizeMetadataQueries, "SELECT MIN(status), MAX(ds) FROM test_metadata_aggregation_folding_with_two_partitions_columns WHERE ds BETWEEN '2020-07-01' AND '2020-07-02'", anyTree(project(ImmutableMap.of("min", expression("'A'"), "max", expression("'2020-07-02'")), anyTree(values()))));
        // All matching partitions have stats. Enable rewrite.
        assertPlan(optimizeMetadataQueries, "SELECT MIN(ds) ds, MIN(status) status FROM test_metadata_aggregation_folding_with_two_partitions_columns", anyTree(project(ImmutableMap.of("ds", expression("'2020-07-01'"), "status", expression("'A'")), anyTree(values()))));
        // Resulting partition doesn't have stats. Disable rewrite.
        assertPlan(optimizeMetadataQueries, "SELECT MAX(status) status FROM test_metadata_aggregation_folding_with_two_partitions_columns", anyTree(tableScanWithConstraint("test_metadata_aggregation_folding_with_two_partitions_columns", ImmutableMap.of("status", multipleValues(VarcharType.createVarcharType(1), utf8Slices("A", "B", "C")), "ds", multipleValues(VARCHAR, utf8Slices("2020-07-01", "2020-07-02", "2020-07-03"))))));
    } finally {
        queryRunner.execute("DROP TABLE IF EXISTS test_metadata_aggregation_folding_with_two_partitions_columns");
    }
}

Example 93 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestHiveDistributedJoinQueriesWithDynamicFiltering method testJoinWithSelectiveBuildSide.

@Test
public void testJoinWithSelectiveBuildSide() {
    Session session = Session.builder(getSession()).setSystemProperty(JOIN_DISTRIBUTION_TYPE, FeaturesConfig.JoinDistributionType.BROADCAST.name()).setSystemProperty(PUSHDOWN_SUBFIELDS_ENABLED, "false").setCatalogSessionProperty(HIVE_CATALOG, PUSHDOWN_FILTER_ENABLED, "false").build();
    DistributedQueryRunner runner = (DistributedQueryRunner) getQueryRunner();
    ResultWithQueryId<MaterializedResult> result = runner.executeWithQueryId(session, "SELECT * FROM lineitem JOIN orders ON lineitem.orderkey = orders.orderkey AND orders.custkey = 1");
    assertGreaterThan(result.getResult().getRowCount(), 0);
    OperatorStats probeStats = searchScanFilterAndProjectOperatorStats(result.getQueryId(), "lineitem");
    // Probe side may be partially scanned, depending on the drivers' scheduling:
    assertLessThanOrEqual(probeStats.getInputPositions(), countRows("lineitem"));
}

Example 94 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class KuduQueryRunnerFactory method createKuduQueryRunnerTpch.

public static QueryRunner createKuduQueryRunnerTpch(Iterable<TpchTable<?>> tables) throws Exception {
    DistributedQueryRunner runner = null;
    String kuduSchema = isSchemaEmulationEnabled() ? "tpch" : "default";
    try {
        runner = DistributedQueryRunner.builder(createSession(kuduSchema)).setNodeCount(3).build();
        runner.installPlugin(new TpchPlugin());
        runner.createCatalog("tpch", "tpch");
        installKuduConnector(runner, kuduSchema);
        copyTpchTables(runner, "tpch", TINY_SCHEMA_NAME, createSession(kuduSchema), tables);
        return runner;
    } catch (Throwable e) {
        closeAllSuppress(e, runner);
        throw e;
    }
}

Example 95 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class PrometheusQueryRunner method createPrometheusQueryRunner.

public static DistributedQueryRunner createPrometheusQueryRunner(PrometheusServer server) throws Exception {
    DistributedQueryRunner queryRunner = null;
    try {
        queryRunner = DistributedQueryRunner.builder(createSession()).build();
        queryRunner.installPlugin(new PrometheusPlugin());
        Map<String, String> properties = ImmutableMap.of("prometheus.uri", server.getUri().toString());
        queryRunner.createCatalog("prometheus", "prometheus", properties);
        return queryRunner;
    } catch (Throwable e) {
        closeAllSuppress(e, queryRunner);
        throw e;
    }
}