Examples with ConnectorTableHandle com.facebook.presto.spi.ConnectorTableHandle used on opensource projects

Example 56 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class AbstractTestHiveClient method testGetPartitionSplitsTableNotReadablePartition.

@Test
public void testGetPartitionSplitsTableNotReadablePartition() {
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableNotReadable);
        assertNotNull(tableHandle);
        ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds");
        assertNotNull(dsColumn);
        ConnectorTableLayout tableLayout = getTableLayout(session, metadata, tableHandle, Constraint.alwaysTrue(), transaction);
        try {
            getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, tableLayout.getHandle(), SPLIT_SCHEDULING_CONTEXT));
            fail("Expected HiveNotReadableException");
        } catch (HiveNotReadableException e) {
            assertThat(e).hasMessageMatching("Table '.*\\.presto_test_not_readable' is not readable: reason for not readable");
            assertEquals(e.getTableName(), tableNotReadable);
            assertEquals(e.getPartition(), Optional.empty());
        }
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession(ImmutableMap.of(OFFLINE_DATA_DEBUG_MODE_ENABLED, true));
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableNotReadable);
        assertNotNull(tableHandle);
        ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds");
        assertNotNull(dsColumn);
        ConnectorTableLayout tableLayout = getTableLayout(session, metadata, tableHandle, Constraint.alwaysTrue(), transaction);
        getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, tableLayout.getHandle(), SPLIT_SCHEDULING_CONTEXT));
    }
}

Example 57 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class AbstractTestHiveClient method getTableHandle.

protected ConnectorTableHandle getTableHandle(ConnectorMetadata metadata, SchemaTableName tableName) {
    ConnectorTableHandle handle = metadata.getTableHandle(newSession(), tableName);
    checkArgument(handle != null, "table not found: %s", tableName);
    return handle;
}

Example 58 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class FunctionAssertions method compileScanFilterProject.

private static SourceOperatorFactory compileScanFilterProject(SqlFunctionProperties sqlFunctionProperties, Optional<RowExpression> filter, RowExpression projection, ExpressionCompiler compiler) {
    try {
        Supplier<CursorProcessor> cursorProcessor = compiler.compileCursorProcessor(sqlFunctionProperties, filter, ImmutableList.of(projection), SOURCE_ID);
        Supplier<PageProcessor> pageProcessor = compiler.compilePageProcessor(sqlFunctionProperties, filter, ImmutableList.of(projection));
        return new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), SOURCE_ID, PAGE_SOURCE_PROVIDER, cursorProcessor, pageProcessor, new TableHandle(new ConnectorId("test"), new ConnectorTableHandle() {
        }, new ConnectorTransactionHandle() {
        }, Optional.empty()), ImmutableList.of(), ImmutableList.of(projection.getType()), Optional.empty(), new DataSize(0, BYTE), 0);
    } catch (Throwable e) {
        if (e instanceof UncheckedExecutionException) {
            e = e.getCause();
        }
        throw new RuntimeException("Error compiling filter " + filter + ": " + e.getMessage(), e);
    }
}

Example 59 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class HiveMetadata method getTableStatistics.

@Override
public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Optional<ConnectorTableLayoutHandle> tableLayoutHandle, List<ColumnHandle> columnHandles, Constraint<ColumnHandle> constraint) {
    if (!isStatisticsEnabled(session)) {
        return TableStatistics.empty();
    }
    if (!tableLayoutHandle.isPresent() || !((HiveTableLayoutHandle) tableLayoutHandle.get()).isPushdownFilterEnabled()) {
        Map<String, ColumnHandle> columns = columnHandles.stream().map(HiveColumnHandle.class::cast).filter(not(HiveColumnHandle::isHidden)).collect(toImmutableMap(HiveColumnHandle::getName, Function.identity()));
        Map<String, Type> columnTypes = columns.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> getColumnMetadata(session, tableHandle, entry.getValue()).getType()));
        List<HivePartition> partitions = partitionManager.getPartitions(metastore, tableHandle, constraint, session).getPartitions();
        return hiveStatisticsProvider.getTableStatistics(session, ((HiveTableHandle) tableHandle).getSchemaTableName(), columns, columnTypes, partitions);
    }
    verify(!constraint.predicate().isPresent());
    HiveTableLayoutHandle hiveLayoutHandle = (HiveTableLayoutHandle) tableLayoutHandle.get();
    Set<String> columnNames = columnHandles.stream().map(HiveColumnHandle.class::cast).map(HiveColumnHandle::getName).collect(toImmutableSet());
    Set<ColumnHandle> allColumnHandles = ImmutableSet.<ColumnHandle>builder().addAll(columnHandles).addAll(hiveLayoutHandle.getPredicateColumns().values().stream().filter(column -> !columnNames.contains(column.getName())).collect(toImmutableList())).build();
    Map<String, ColumnHandle> allColumns = Maps.uniqueIndex(allColumnHandles, column -> ((HiveColumnHandle) column).getName());
    Map<String, Type> allColumnTypes = allColumns.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> getColumnMetadata(session, tableHandle, entry.getValue()).getType()));
    Constraint<ColumnHandle> combinedConstraint = new Constraint<>(constraint.getSummary().intersect(hiveLayoutHandle.getDomainPredicate().transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null).transform(allColumns::get)));
    SubfieldExtractor subfieldExtractor = new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session);
    RowExpression domainPredicate = rowExpressionService.getDomainTranslator().toPredicate(hiveLayoutHandle.getDomainPredicate().transform(subfield -> subfieldExtractor.toRowExpression(subfield, allColumnTypes.get(subfield.getRootName()))));
    RowExpression combinedPredicate = binaryExpression(SpecialFormExpression.Form.AND, ImmutableList.of(hiveLayoutHandle.getRemainingPredicate(), domainPredicate));
    List<HivePartition> partitions = partitionManager.getPartitions(metastore, tableHandle, combinedConstraint, session).getPartitions();
    TableStatistics tableStatistics = hiveStatisticsProvider.getTableStatistics(session, ((HiveTableHandle) tableHandle).getSchemaTableName(), allColumns, allColumnTypes, partitions);
    return filterStatsCalculatorService.filterStats(tableStatistics, combinedPredicate, session, ImmutableBiMap.copyOf(allColumns).inverse(), allColumnTypes);
}

Example 60 with ConnectorTableHandle

use of com.facebook.presto.spi.ConnectorTableHandle in project presto by prestodb.

the class HiveMetadata method createTemporaryTable.

@Override
public ConnectorTableHandle createTemporaryTable(ConnectorSession session, List<ColumnMetadata> columns, Optional<ConnectorPartitioningMetadata> partitioningMetadata) {
    String schemaName = getTemporaryTableSchema(session);
    HiveStorageFormat storageFormat = getTemporaryTableStorageFormat(session);
    Optional<HiveBucketProperty> bucketProperty = partitioningMetadata.map(partitioning -> {
        Set<String> allColumns = columns.stream().map(ColumnMetadata::getName).collect(toImmutableSet());
        if (!allColumns.containsAll(partitioning.getPartitionColumns())) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, format("Bucketing columns %s not present in schema", Sets.difference(ImmutableSet.copyOf(partitioning.getPartitionColumns()), allColumns)));
        }
        HivePartitioningHandle partitioningHandle = (HivePartitioningHandle) partitioning.getPartitioningHandle();
        List<String> partitionColumns = partitioning.getPartitionColumns();
        BucketFunctionType bucketFunctionType = partitioningHandle.getBucketFunctionType();
        switch(bucketFunctionType) {
            case HIVE_COMPATIBLE:
                return new HiveBucketProperty(partitionColumns, partitioningHandle.getBucketCount(), ImmutableList.of(), HIVE_COMPATIBLE, Optional.empty());
            case PRESTO_NATIVE:
                Map<String, Type> columnNameToTypeMap = columns.stream().collect(toMap(ColumnMetadata::getName, ColumnMetadata::getType));
                return new HiveBucketProperty(partitionColumns, partitioningHandle.getBucketCount(), ImmutableList.of(), PRESTO_NATIVE, Optional.of(partitionColumns.stream().map(columnNameToTypeMap::get).collect(toImmutableList())));
            default:
                throw new IllegalArgumentException("Unsupported bucket function type " + bucketFunctionType);
        }
    });
    if (isUsePageFileForHiveUnsupportedType(session)) {
        if (!columns.stream().map(ColumnMetadata::getType).allMatch(HiveTypeTranslator::isSupportedHiveType)) {
            storageFormat = PAGEFILE;
        }
    }
    // PAGEFILE format doesn't require translation to hive type,
    // choose HIVE_BINARY as a default hive type to make it compatible with Hive connector
    Optional<HiveType> defaultHiveType = storageFormat == PAGEFILE ? Optional.of(HIVE_BINARY) : Optional.empty();
    List<HiveColumnHandle> columnHandles = getColumnHandles(// type to the boolean type that is binary compatible
    translateHiveUnsupportedTypesForTemporaryTable(columns, typeManager), ImmutableSet.of(), typeTranslator, defaultHiveType);
    validateColumns(storageFormat, columnHandles);
    HiveStorageFormat finalStorageFormat = storageFormat;
    String tableName = PRESTO_TEMPORARY_TABLE_NAME_PREFIX + finalStorageFormat.name() + "_" + session.getQueryId().replaceAll("-", "_") + "_" + randomUUID().toString().replaceAll("-", "_");
    Table table = Table.builder().setDatabaseName(schemaName).setTableName(tableName).setOwner(session.getUser()).setTableType(TEMPORARY_TABLE).setDataColumns(columnHandles.stream().map(handle -> new Column(handle.getName(), handle.getHiveType(), handle.getComment(), Optional.empty())).collect(toImmutableList())).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(finalStorageFormat)).setBucketProperty(bucketProperty).setLocation("")).build();
    List<String> partitionColumnNames = table.getPartitionColumns().stream().map(Column::getName).collect(toImmutableList());
    List<HiveColumnHandle> hiveColumnHandles = hiveColumnHandles(table);
    Map<String, Type> columnTypes = hiveColumnHandles.stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager)));
    Map<String, Set<ColumnStatisticType>> columnStatisticTypes = hiveColumnHandles.stream().filter(columnHandle -> !partitionColumnNames.contains(columnHandle.getName())).filter(column -> !column.isHidden()).collect(toImmutableMap(HiveColumnHandle::getName, column -> ImmutableSet.copyOf(getSupportedColumnStatisticsForTemporaryTable(typeManager.getType(column.getTypeSignature())))));
    metastore.createTable(session, table, buildInitialPrivilegeSet(table.getOwner()), Optional.empty(), false, createEmptyPartitionStatistics(columnTypes, columnStatisticTypes));
    return new HiveTableHandle(schemaName, tableName);
}