Examples with PrestoTableType com.facebook.presto.hive.metastore.PrestoTableType used on opensource projects

Example 1 with PrestoTableType

use of com.facebook.presto.hive.metastore.PrestoTableType in project presto by prestodb.

the class HiveMetadata method prepareTable.

private Table prepareTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, PrestoTableType tableType) {
    SchemaTableName schemaTableName = tableMetadata.getTable();
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties());
    Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties());
    if ((bucketProperty.isPresent() || !partitionedBy.isEmpty()) && getAvroSchemaUrl(tableMetadata.getProperties()) != null) {
        throw new PrestoException(NOT_SUPPORTED, "Bucketing/Partitioning columns not supported when Avro schema url is set");
    }
    List<HiveColumnHandle> columnHandles = getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator);
    HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties());
    List<SortingColumn> preferredOrderingColumns = getPreferredOrderingColumns(tableMetadata.getProperties());
    Optional<TableEncryptionProperties> tableEncryptionProperties = getTableEncryptionPropertiesFromTableProperties(tableMetadata, hiveStorageFormat, partitionedBy);
    if (tableEncryptionProperties.isPresent() && partitionedBy.isEmpty()) {
        throw new PrestoException(HIVE_UNSUPPORTED_ENCRYPTION_OPERATION, "Creating an encrypted table without partitions is not supported. Use CREATE TABLE AS SELECT to " + "create an encrypted table without partitions");
    }
    validateColumns(hiveStorageFormat, columnHandles);
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Map<String, HiveColumnHandle> columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName);
    List<Column> partitionColumns = partitionedBy.stream().map(columnHandlesByName::get).map(columnHandle -> columnHandleToColumn(metastoreContext, columnHandle)).collect(toList());
    checkPartitionTypesSupported(partitionColumns);
    Path targetPath;
    if (tableType.equals(EXTERNAL_TABLE)) {
        if (!createsOfNonManagedTablesEnabled) {
            throw new PrestoException(NOT_SUPPORTED, "Cannot create non-managed Hive table");
        }
        String externalLocation = getExternalLocation(tableMetadata.getProperties());
        targetPath = getExternalPath(new HdfsContext(session, schemaName, tableName, externalLocation, true), externalLocation);
    } else if (tableType.equals(MANAGED_TABLE) || tableType.equals(MATERIALIZED_VIEW)) {
        LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, isTempPathRequired(session, bucketProperty, preferredOrderingColumns));
        targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
    } else {
        throw new IllegalStateException(format("%s is not a valid table type to be created.", tableType));
    }
    Map<String, String> tableProperties = getEmptyTableProperties(tableMetadata, new HdfsContext(session, schemaName, tableName, targetPath.toString(), true), hiveStorageFormat, tableEncryptionProperties);
    return buildTableObject(session.getQueryId(), schemaName, tableName, session.getUser(), columnHandles, hiveStorageFormat, partitionedBy, bucketProperty, preferredOrderingColumns, tableProperties, targetPath, tableType, prestoVersion, metastoreContext);
}

Example 2 with PrestoTableType

use of com.facebook.presto.hive.metastore.PrestoTableType in project presto by prestodb.

the class ThriftMetastoreUtil method toMetastoreApiTable.

public static org.apache.hadoop.hive.metastore.api.Table toMetastoreApiTable(Table table, PrincipalPrivileges privileges, ColumnConverter columnConverter) {
    org.apache.hadoop.hive.metastore.api.Table result = new org.apache.hadoop.hive.metastore.api.Table();
    result.setDbName(table.getDatabaseName());
    result.setTableName(table.getTableName());
    result.setOwner(table.getOwner());
    PrestoTableType tableType = table.getTableType();
    checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW, MATERIALIZED_VIEW).contains(tableType), "Invalid table type: %s", table.getTableType());
    // TableType.MATERIALIZED_VIEW is not supported by Hive metastore until Hive 3.0. Use MANAGED_TABLE for now.
    if (MATERIALIZED_VIEW.equals(tableType)) {
        tableType = MANAGED_TABLE;
    }
    result.setTableType(tableType.name());
    result.setParameters(table.getParameters());
    result.setPartitionKeys(table.getPartitionColumns().stream().map(col -> ThriftMetastoreUtil.toMetastoreApiFieldSchema(col, columnConverter)).collect(toList()));
    result.setSd(makeStorageDescriptor(table.getTableName(), table.getDataColumns(), table.getStorage(), columnConverter));
    result.setPrivileges(toMetastoreApiPrincipalPrivilegeSet(privileges));
    result.setViewOriginalText(table.getViewOriginalText().orElse(null));
    result.setViewExpandedText(table.getViewExpandedText().orElse(null));
    return result;
}

Example 3 with PrestoTableType

use of com.facebook.presto.hive.metastore.PrestoTableType in project presto by prestodb.

the class ThriftMetastoreUtil method fromMetastoreApiTable.

public static Table fromMetastoreApiTable(org.apache.hadoop.hive.metastore.api.Table table, List<FieldSchema> schema, ColumnConverter columnConverter) {
    StorageDescriptor storageDescriptor = table.getSd();
    if (storageDescriptor == null) {
        throw new PrestoException(HIVE_INVALID_METADATA, "Table is missing storage descriptor");
    }
    // TODO: remove the table type change after Hive 3.0 update.
    // Materialized view fetched from Hive Metastore uses TableType.MANAGED_TABLE before Hive 3.0. Cast it back to MATERIALIZED_VIEW here.
    PrestoTableType tableType = PrestoTableType.optionalValueOf(table.getTableType()).orElse(OTHER);
    if (table.getParameters() != null && "true".equals(table.getParameters().get(PRESTO_MATERIALIZED_VIEW_FLAG))) {
        checkState(TableType.MANAGED_TABLE.name().equals(table.getTableType()), "Materialized view %s has incorrect table type %s from Metastore.", table.getTableName(), table.getTableType());
        tableType = MATERIALIZED_VIEW;
    }
    Table.Builder tableBuilder = Table.builder().setDatabaseName(table.getDbName()).setTableName(table.getTableName()).setOwner(nullToEmpty(table.getOwner())).setTableType(tableType).setDataColumns(schema.stream().map(columnConverter::toColumn).collect(toList())).setPartitionColumns(table.getPartitionKeys().stream().map(columnConverter::toColumn).collect(toList())).setParameters(table.getParameters() == null ? ImmutableMap.of() : table.getParameters()).setViewOriginalText(Optional.ofNullable(emptyToNull(table.getViewOriginalText()))).setViewExpandedText(Optional.ofNullable(emptyToNull(table.getViewExpandedText())));
    fromMetastoreApiStorageDescriptor(storageDescriptor, tableBuilder.getStorageBuilder(), table.getTableName());
    return tableBuilder.build();
}

Example 4 with PrestoTableType

use of com.facebook.presto.hive.metastore.PrestoTableType in project presto by prestodb.

the class HiveMetadata method buildTableObject.

private static Table buildTableObject(String queryId, String schemaName, String tableName, String tableOwner, List<HiveColumnHandle> columnHandles, HiveStorageFormat hiveStorageFormat, List<String> partitionedBy, Optional<HiveBucketProperty> bucketProperty, List<SortingColumn> preferredOrderingColumns, Map<String, String> additionalTableParameters, Path targetPath, PrestoTableType tableType, String prestoVersion, MetastoreContext metastoreContext) {
    Map<String, HiveColumnHandle> columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName);
    List<Column> partitionColumns = partitionedBy.stream().map(columnHandlesByName::get).map(column -> columnHandleToColumn(metastoreContext, column)).collect(toList());
    Set<String> partitionColumnNames = ImmutableSet.copyOf(partitionedBy);
    ImmutableList.Builder<Column> columns = ImmutableList.builder();
    for (HiveColumnHandle columnHandle : columnHandles) {
        String name = columnHandle.getName();
        HiveType type = columnHandle.getHiveType();
        if (!partitionColumnNames.contains(name)) {
            verify(!columnHandle.isPartitionKey(), "Column handles are not consistent with partitioned by property");
            columns.add(columnHandleToColumn(metastoreContext, columnHandle));
        } else {
            verify(columnHandle.isPartitionKey(), "Column handles are not consistent with partitioned by property");
        }
    }
    ImmutableMap.Builder<String, String> tableParameters = ImmutableMap.<String, String>builder().put(PRESTO_VERSION_NAME, prestoVersion).put(PRESTO_QUERY_ID_NAME, queryId).putAll(additionalTableParameters);
    if (tableType.equals(EXTERNAL_TABLE)) {
        tableParameters.put("EXTERNAL", "TRUE");
    }
    Table.Builder tableBuilder = Table.builder().setDatabaseName(schemaName).setTableName(tableName).setOwner(tableOwner).setTableType(tableType).setDataColumns(columns.build()).setPartitionColumns(partitionColumns).setParameters(tableParameters.build());
    tableBuilder.getStorageBuilder().setStorageFormat(fromHiveStorageFormat(hiveStorageFormat)).setBucketProperty(bucketProperty).setParameters(ImmutableMap.of(PREFERRED_ORDERING_COLUMNS, encodePreferredOrderingColumns(preferredOrderingColumns))).setLocation(targetPath.toString());
    return tableBuilder.build();
}

Example 5 with PrestoTableType

use of com.facebook.presto.hive.metastore.PrestoTableType in project presto by prestodb.

the class HiveMetadata method createTable.

@Override
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) {
    PrestoTableType tableType = isExternalTable(tableMetadata.getProperties()) ? EXTERNAL_TABLE : MANAGED_TABLE;
    Table table = prepareTable(session, tableMetadata, tableType);
    PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(table.getOwner());
    HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? createZeroStatistics() : createEmptyStatistics();
    metastore.createTable(session, table, principalPrivileges, Optional.empty(), ignoreExisting, new PartitionStatistics(basicStatistics, ImmutableMap.of()));
}