Examples with ExtendedHiveMetastore com.facebook.presto.hive.metastore.ExtendedHiveMetastore used on opensource projects

Example 16 with ExtendedHiveMetastore

use of com.facebook.presto.hive.metastore.ExtendedHiveMetastore in project presto by prestodb.

the class TestHivePageSink method writeTestFile.

private static long writeTestFile(HiveClientConfig config, MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore metastore, String outputPath) {
    HiveTransactionHandle transaction = new HiveTransactionHandle();
    HiveWriterStats stats = new HiveWriterStats();
    ConnectorPageSink pageSink = createPageSink(transaction, config, metastoreClientConfig, metastore, new Path("file:///" + outputPath), stats);
    List<LineItemColumn> columns = getTestColumns();
    List<Type> columnTypes = columns.stream().map(LineItemColumn::getType).map(TestHivePageSink::getHiveType).map(hiveType -> hiveType.getType(FUNCTION_AND_TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    int rows = 0;
    for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) {
        rows++;
        if (rows >= NUM_ROWS) {
            break;
        }
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            LineItemColumn column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(lineItem));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(lineItem));
                    break;
                case DATE:
                    DATE.writeLong(blockBuilder, column.getDate(lineItem));
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(lineItem));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(lineItem)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
    }
    Page page = pageBuilder.build();
    pageSink.appendPage(page);
    getFutureValue(pageSink.finish());
    File outputDir = new File(outputPath);
    List<File> files = ImmutableList.copyOf(outputDir.listFiles((dir, name) -> !name.endsWith(".crc")));
    File outputFile = getOnlyElement(files);
    long length = outputFile.length();
    ConnectorPageSource pageSource = createPageSource(transaction, config, metastoreClientConfig, outputFile);
    List<Page> pages = new ArrayList<>();
    while (!pageSource.isFinished()) {
        Page nextPage = pageSource.getNextPage();
        if (nextPage != null) {
            pages.add(nextPage.getLoadedPage());
        }
    }
    MaterializedResult expectedResults = toMaterializedResult(getSession(config), columnTypes, ImmutableList.of(page));
    MaterializedResult results = toMaterializedResult(getSession(config), columnTypes, pages);
    assertEquals(results, expectedResults);
    assertEquals(stats.getInputPageSizeInBytes().getAllTime().getMax(), page.getRetainedSizeInBytes());
    return length;
}

Example 17 with ExtendedHiveMetastore

use of com.facebook.presto.hive.metastore.ExtendedHiveMetastore in project presto by prestodb.

the class AbstractTestHiveClient method testDropColumn.

@Test
public void testDropColumn() throws Exception {
    SchemaTableName tableName = temporaryTable("test_drop_column");
    try {
        doCreateEmptyTable(tableName, ORC, CREATE_TABLE_COLUMNS);
        ExtendedHiveMetastore metastoreClient = getMetastoreClient();
        metastoreClient.dropColumn(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), CREATE_TABLE_COLUMNS.get(0).getName());
        Optional<Table> table = metastoreClient.getTable(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName());
        assertTrue(table.isPresent());
        List<Column> columns = table.get().getDataColumns();
        assertEquals(columns.get(0).getName(), CREATE_TABLE_COLUMNS.get(1).getName());
        assertFalse(columns.stream().map(Column::getName).anyMatch(colName -> colName.equals(CREATE_TABLE_COLUMNS.get(0).getName())));
    } finally {
        dropTable(tableName);
    }
}

Example 18 with ExtendedHiveMetastore

use of com.facebook.presto.hive.metastore.ExtendedHiveMetastore in project presto by prestodb.

the class AbstractTestHiveClient method testUpdateTableStatistics.

protected void testUpdateTableStatistics(SchemaTableName tableName, PartitionStatistics initialStatistics, PartitionStatistics... statistics) {
    ExtendedHiveMetastore metastoreClient = getMetastoreClient();
    assertThat(metastoreClient.getTableStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName())).isEqualTo(initialStatistics);
    AtomicReference<PartitionStatistics> expectedStatistics = new AtomicReference<>(initialStatistics);
    for (PartitionStatistics partitionStatistics : statistics) {
        metastoreClient.updateTableStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), actualStatistics -> {
            assertThat(actualStatistics).isEqualTo(expectedStatistics.get());
            return partitionStatistics;
        });
        assertThat(metastoreClient.getTableStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName())).isEqualTo(partitionStatistics);
        expectedStatistics.set(partitionStatistics);
    }
    assertThat(metastoreClient.getTableStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName())).isEqualTo(expectedStatistics.get());
    metastoreClient.updateTableStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), actualStatistics -> {
        assertThat(actualStatistics).isEqualTo(expectedStatistics.get());
        return initialStatistics;
    });
    assertThat(metastoreClient.getTableStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName())).isEqualTo(initialStatistics);
}

Example 19 with ExtendedHiveMetastore

use of com.facebook.presto.hive.metastore.ExtendedHiveMetastore in project presto by prestodb.

the class AbstractTestHiveClient method testAddColumn.

@Test
public void testAddColumn() throws Exception {
    SchemaTableName tableName = temporaryTable("test_add_column");
    try {
        doCreateEmptyTable(tableName, ORC, CREATE_TABLE_COLUMNS);
        ExtendedHiveMetastore metastoreClient = getMetastoreClient();
        metastoreClient.addColumn(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), "new_col", HIVE_LONG, null);
        Optional<Table> table = metastoreClient.getTable(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName());
        assertTrue(table.isPresent());
        List<Column> columns = table.get().getDataColumns();
        assertEquals(columns.get(columns.size() - 1).getName(), "new_col");
    } finally {
        dropTable(tableName);
    }
}

Example 20 with ExtendedHiveMetastore

use of com.facebook.presto.hive.metastore.ExtendedHiveMetastore in project presto by prestodb.

the class AbstractTestHiveClient method eraseStatistics.

private void eraseStatistics(SchemaTableName schemaTableName) {
    ExtendedHiveMetastore metastoreClient = getMetastoreClient();
    metastoreClient.updateTableStatistics(METASTORE_CONTEXT, schemaTableName.getSchemaName(), schemaTableName.getTableName(), statistics -> new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of()));
    Table table = metastoreClient.getTable(METASTORE_CONTEXT, schemaTableName.getSchemaName(), schemaTableName.getTableName()).orElseThrow(() -> new TableNotFoundException(schemaTableName));
    List<String> partitionColumns = table.getPartitionColumns().stream().map(Column::getName).collect(toImmutableList());
    if (!table.getPartitionColumns().isEmpty()) {
        List<String> partitionNames = metastoreClient.getPartitionNames(METASTORE_CONTEXT, schemaTableName.getSchemaName(), schemaTableName.getTableName()).orElse(ImmutableList.of());
        List<Partition> partitions = metastoreClient.getPartitionsByNames(METASTORE_CONTEXT, schemaTableName.getSchemaName(), schemaTableName.getTableName(), partitionNames).entrySet().stream().map(Map.Entry::getValue).filter(Optional::isPresent).map(Optional::get).collect(toImmutableList());
        for (Partition partition : partitions) {
            metastoreClient.updatePartitionStatistics(METASTORE_CONTEXT, schemaTableName.getSchemaName(), schemaTableName.getTableName(), makePartName(partitionColumns, partition.getValues()), statistics -> new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of()));
        }
    }
}