Examples with ExtendedHiveMetastore com.facebook.presto.hive.metastore.ExtendedHiveMetastore used on opensource projects

Example 6 with ExtendedHiveMetastore

use of com.facebook.presto.hive.metastore.ExtendedHiveMetastore in project presto by prestodb.

the class AbstractTestHiveClient method setup.

protected final void setup(String host, int port, String databaseName, String timeZone) {
    HiveClientConfig hiveClientConfig = getHiveClientConfig();
    CacheConfig cacheConfig = getCacheConfig();
    MetastoreClientConfig metastoreClientConfig = getMetastoreClientConfig();
    hiveClientConfig.setTimeZone(timeZone);
    String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
    if (proxy != null) {
        metastoreClientConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
    }
    HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, host, port);
    ExtendedHiveMetastore metastore = new CachingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster, metastoreClientConfig), new HivePartitionMutator()), executor, false, Duration.valueOf("1m"), Duration.valueOf("15s"), 10000, false, MetastoreCacheScope.ALL, 0.0);
    setup(databaseName, hiveClientConfig, cacheConfig, metastoreClientConfig, metastore);
}

Example 7 with ExtendedHiveMetastore

use of com.facebook.presto.hive.metastore.ExtendedHiveMetastore in project presto by prestodb.

the class HiveBenchmarkQueryRunner method createLocalQueryRunner.

public static LocalQueryRunner createLocalQueryRunner(File tempDir) {
    Session session = testSessionBuilder().setCatalog("hive").setSchema("tpch").build();
    LocalQueryRunner localQueryRunner = new LocalQueryRunner(session);
    // add tpch
    localQueryRunner.createCatalog("tpch", new TpchConnectorFactory(1), ImmutableMap.of());
    // add hive
    File hiveDir = new File(tempDir, "hive_data");
    ExtendedHiveMetastore metastore = createTestingFileHiveMetastore(hiveDir);
    metastore.createDatabase(METASTORE_CONTEXT, Database.builder().setDatabaseName("tpch").setOwnerName("public").setOwnerType(PrincipalType.ROLE).build());
    HiveConnectorFactory hiveConnectorFactory = new HiveConnectorFactory("hive", HiveBenchmarkQueryRunner.class.getClassLoader(), Optional.of(metastore));
    Map<String, String> hiveCatalogConfig = ImmutableMap.<String, String>builder().put("hive.max-split-size", "10GB").build();
    localQueryRunner.createCatalog("hive", hiveConnectorFactory, hiveCatalogConfig);
    localQueryRunner.execute("CREATE TABLE orders AS SELECT * FROM tpch.sf1.orders");
    localQueryRunner.execute("CREATE TABLE lineitem AS SELECT * FROM tpch.sf1.lineitem");
    return localQueryRunner;
}

Example 8 with ExtendedHiveMetastore

use of com.facebook.presto.hive.metastore.ExtendedHiveMetastore in project presto by prestodb.

the class AbstractTestHiveClientLocal method initialize.

@BeforeClass
public void initialize() {
    tempDir = Files.createTempDir();
    ExtendedHiveMetastore metastore = createMetastore(tempDir);
    metastore.createDatabase(METASTORE_CONTEXT, Database.builder().setDatabaseName(testDbName).setOwnerName("public").setOwnerType(PrincipalType.ROLE).build());
    HiveClientConfig hiveConfig = new HiveClientConfig().setTimeZone("America/Los_Angeles");
    MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig();
    setup(testDbName, hiveConfig, new CacheConfig(), metastoreClientConfig, metastore);
}

Example 9 with ExtendedHiveMetastore

use of com.facebook.presto.hive.metastore.ExtendedHiveMetastore in project presto by prestodb.

the class TestHiveLogicalPlanner method testMetadataAggregationFoldingWithEmptyPartitionsAndMetastoreThreshold.

@Test
public void testMetadataAggregationFoldingWithEmptyPartitionsAndMetastoreThreshold() {
    QueryRunner queryRunner = getQueryRunner();
    Session optimizeMetadataQueriesWithHighThreshold = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(OPTIMIZE_METADATA_QUERIES, Boolean.toString(true)).setSystemProperty(OPTIMIZE_METADATA_QUERIES_CALL_THRESHOLD, Integer.toString(100)).build();
    Session optimizeMetadataQueriesWithLowThreshold = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(OPTIMIZE_METADATA_QUERIES, Boolean.toString(true)).setSystemProperty(OPTIMIZE_METADATA_QUERIES_CALL_THRESHOLD, Integer.toString(1)).build();
    Session optimizeMetadataQueriesIgnoreStatsWithLowThreshold = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(OPTIMIZE_METADATA_QUERIES_IGNORE_STATS, Boolean.toString(true)).setSystemProperty(OPTIMIZE_METADATA_QUERIES_CALL_THRESHOLD, Integer.toString(1)).build();
    Session shufflePartitionColumns = Session.builder(this.getQueryRunner().getDefaultSession()).setCatalogSessionProperty(HIVE_CATALOG, SHUFFLE_PARTITIONED_COLUMNS_FOR_TABLE_WRITE, Boolean.toString(true)).build();
    queryRunner.execute(shufflePartitionColumns, "CREATE TABLE test_metadata_aggregation_folding_with_empty_partitions_with_threshold WITH (partitioned_by = ARRAY['ds']) AS " + "SELECT orderkey, CAST(to_iso8601(date_add('DAY', orderkey % 2, date('2020-07-01'))) AS VARCHAR) AS ds FROM orders WHERE orderkey < 1000");
    ExtendedHiveMetastore metastore = replicateHiveMetastore((DistributedQueryRunner) queryRunner);
    MetastoreContext metastoreContext = new MetastoreContext(getSession().getUser(), getSession().getQueryId().getId(), Optional.empty(), Optional.empty(), Optional.empty(), false, HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER);
    Table table = metastore.getTable(metastoreContext, getSession().getSchema().get(), "test_metadata_aggregation_folding_with_empty_partitions_with_threshold").get();
    // Add one partition with no statistics.
    String partitionNameNoStats = "ds=2020-07-20";
    Partition partitionNoStats = createDummyPartition(table, partitionNameNoStats);
    metastore.addPartitions(metastoreContext, table.getDatabaseName(), table.getTableName(), ImmutableList.of(new PartitionWithStatistics(partitionNoStats, partitionNameNoStats, PartitionStatistics.empty())));
    // Add one partition with statistics indicating that it has no rows.
    String emptyPartitionName = "ds=2020-06-30";
    Partition emptyPartition = createDummyPartition(table, emptyPartitionName);
    metastore.addPartitions(metastoreContext, table.getDatabaseName(), table.getTableName(), ImmutableList.of(new PartitionWithStatistics(emptyPartition, emptyPartitionName, PartitionStatistics.builder().setBasicStatistics(new HiveBasicStatistics(1, 0, 0, 0)).build())));
    try {
        // Test the non-reducible code path.
        // Enable rewrite as all matching partitions have stats.
        assertPlan(optimizeMetadataQueriesWithHighThreshold, "SELECT DISTINCT ds FROM test_metadata_aggregation_folding_with_empty_partitions_with_threshold WHERE ds BETWEEN '2020-07-01' AND '2020-07-03'", anyTree(values(ImmutableList.of("ds"), ImmutableList.of(ImmutableList.of(new StringLiteral("2020-07-01")), ImmutableList.of(new StringLiteral("2020-07-02"))))));
        // All matching partitions have stats, Metastore threshold is reached, Disable rewrite
        assertPlan(optimizeMetadataQueriesWithLowThreshold, "SELECT DISTINCT ds FROM test_metadata_aggregation_folding_with_empty_partitions_with_threshold WHERE ds BETWEEN '2020-07-01' AND '2020-07-03'", anyTree(tableScanWithConstraint("test_metadata_aggregation_folding_with_empty_partitions_with_threshold", ImmutableMap.of("ds", multipleValues(VARCHAR, utf8Slices("2020-07-01", "2020-07-02"))))));
        // All matching partitions have stats, Metastore threshold is reached, but IgnoreStats will overwrite, Enable rewrite
        assertPlan(optimizeMetadataQueriesIgnoreStatsWithLowThreshold, "SELECT DISTINCT ds FROM test_metadata_aggregation_folding_with_empty_partitions_with_threshold WHERE ds BETWEEN '2020-07-01' AND '2020-07-03'", anyTree(values(ImmutableList.of("ds"), ImmutableList.of(ImmutableList.of(new StringLiteral("2020-07-01")), ImmutableList.of(new StringLiteral("2020-07-02"))))));
    } finally {
        queryRunner.execute("DROP TABLE IF EXISTS test_metadata_aggregation_folding_with_empty_partitions_with_threshold");
    }
}

Example 10 with ExtendedHiveMetastore

use of com.facebook.presto.hive.metastore.ExtendedHiveMetastore in project presto by prestodb.

the class TestHiveClientFileMetastore method createDummyPartitionedTable.

private void createDummyPartitionedTable(SchemaTableName tableName, List<ColumnMetadata> columns, Map<String, String> dynamicPartitionParameters) throws Exception {
    doCreateEmptyTable(tableName, ORC, columns);
    ExtendedHiveMetastore metastoreClient = getMetastoreClient();
    Table table = metastoreClient.getTable(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    List<String> firstPartitionValues = ImmutableList.of("2020-01-01");
    List<String> secondPartitionValues = ImmutableList.of("2020-01-02");
    List<String> thirdPartitionValues = ImmutableList.of("2020-01-03");
    String firstPartitionName = makePartName(ImmutableList.of("ds"), firstPartitionValues);
    String secondPartitionName = makePartName(ImmutableList.of("ds"), secondPartitionValues);
    String thirdPartitionName = makePartName(ImmutableList.of("ds"), thirdPartitionValues);
    List<PartitionWithStatistics> partitions = ImmutableList.of(firstPartitionName, secondPartitionName).stream().map(partitionName -> new PartitionWithStatistics(createDummyPartition(table, partitionName), partitionName, PartitionStatistics.empty())).collect(toImmutableList());
    metastoreClient.addPartitions(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), partitions);
    metastoreClient.updatePartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), firstPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS);
    metastoreClient.updatePartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), secondPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS);
    List<PartitionWithStatistics> partitionsNotReadable = ImmutableList.of(thirdPartitionName).stream().map(partitionName -> new PartitionWithStatistics(createDummyPartition(table, partitionName, dynamicPartitionParameters), partitionName, PartitionStatistics.empty())).collect(toImmutableList());
    metastoreClient.addPartitions(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), partitionsNotReadable);
    metastoreClient.updatePartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), thirdPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS);
}