Examples with ConnectorNewTableLayout com.facebook.presto.spi.ConnectorNewTableLayout used on opensource projects

Example 6 with ConnectorNewTableLayout

use of com.facebook.presto.spi.ConnectorNewTableLayout in project presto by prestodb.

the class HiveMetadata method getNewTableLayout.

@Override
public Optional<ConnectorNewTableLayout> getNewTableLayout(ConnectorSession session, ConnectorTableMetadata tableMetadata) {
    validatePartitionColumns(tableMetadata);
    validateBucketColumns(tableMetadata);
    validateCsvColumns(tableMetadata);
    Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties());
    if (!bucketProperty.isPresent()) {
        return Optional.empty();
    }
    checkArgument(bucketProperty.get().getBucketFunctionType().equals(BucketFunctionType.HIVE_COMPATIBLE), "bucketFunctionType is expected to be HIVE_COMPATIBLE, got: %s", bucketProperty.get().getBucketFunctionType());
    if (!bucketProperty.get().getSortedBy().isEmpty() && !isSortedWritingEnabled(session)) {
        throw new PrestoException(NOT_SUPPORTED, "Writing to bucketed sorted Hive tables is disabled");
    }
    List<String> bucketedBy = bucketProperty.get().getBucketedBy();
    Map<String, HiveType> hiveTypeMap = tableMetadata.getColumns().stream().collect(toMap(ColumnMetadata::getName, column -> toHiveType(typeTranslator, column.getType())));
    return Optional.of(new ConnectorNewTableLayout(createHiveCompatiblePartitioningHandle(bucketProperty.get().getBucketCount(), bucketedBy.stream().map(hiveTypeMap::get).collect(toImmutableList()), OptionalInt.of(bucketProperty.get().getBucketCount())), bucketedBy));
}

Example 7 with ConnectorNewTableLayout

use of com.facebook.presto.spi.ConnectorNewTableLayout in project presto by prestodb.

the class HiveMetadata method getInsertLayout.

@Override
public Optional<ConnectorNewTableLayout> getInsertLayout(ConnectorSession session, ConnectorTableHandle tableHandle) {
    HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle;
    SchemaTableName tableName = hiveTableHandle.getSchemaTableName();
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Table table = metastore.getTable(metastoreContext, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    Optional<HiveBucketHandle> hiveBucketHandle = getHiveBucketHandle(table);
    if (!hiveBucketHandle.isPresent()) {
        return Optional.empty();
    }
    HiveBucketProperty bucketProperty = table.getStorage().getBucketProperty().orElseThrow(() -> new NoSuchElementException("Bucket property should be set"));
    if (!bucketProperty.getSortedBy().isEmpty() && !isSortedWritingEnabled(session)) {
        throw new PrestoException(NOT_SUPPORTED, "Writing to bucketed sorted Hive tables is disabled");
    }
    HivePartitioningHandle partitioningHandle;
    int bucketCount = hiveBucketHandle.get().getTableBucketCount();
    OptionalInt maxCompatibleBucketCount = OptionalInt.of(bucketCount);
    switch(bucketProperty.getBucketFunctionType()) {
        case HIVE_COMPATIBLE:
            partitioningHandle = createHiveCompatiblePartitioningHandle(bucketCount, hiveBucketHandle.get().getColumns().stream().map(HiveColumnHandle::getHiveType).collect(toImmutableList()), maxCompatibleBucketCount);
            break;
        case PRESTO_NATIVE:
            partitioningHandle = createPrestoNativePartitioningHandle(bucketCount, bucketProperty.getTypes().get(), maxCompatibleBucketCount);
            break;
        default:
            throw new IllegalArgumentException("Unsupported bucket function type " + bucketProperty.getBucketFunctionType());
    }
    List<String> partitionColumns = hiveBucketHandle.get().getColumns().stream().map(HiveColumnHandle::getName).collect(toList());
    return Optional.of(new ConnectorNewTableLayout(partitioningHandle, partitionColumns));
}

Example 8 with ConnectorNewTableLayout

use of com.facebook.presto.spi.ConnectorNewTableLayout in project presto by prestodb.

the class HiveMetadata method beginCreateTable.

@Override
public HiveOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout) {
    verifyJvmTimeZone();
    if (getExternalLocation(tableMetadata.getProperties()) != null) {
        throw new PrestoException(NOT_SUPPORTED, "External tables cannot be created using CREATE TABLE AS");
    }
    if (getAvroSchemaUrl(tableMetadata.getProperties()) != null) {
        throw new PrestoException(NOT_SUPPORTED, "CREATE TABLE AS not supported when Avro schema url is set");
    }
    HiveStorageFormat tableStorageFormat = getHiveStorageFormat(tableMetadata.getProperties());
    List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties());
    Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties());
    List<SortingColumn> preferredOrderingColumns = getPreferredOrderingColumns(tableMetadata.getProperties());
    // get the root directory for the database
    SchemaTableName schemaTableName = tableMetadata.getTable();
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    Optional<TableEncryptionProperties> tableEncryptionProperties = getTableEncryptionPropertiesFromTableProperties(tableMetadata, tableStorageFormat, partitionedBy);
    List<HiveColumnHandle> columnHandles = getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator);
    HiveStorageFormat partitionStorageFormat = isRespectTableFormat(session) ? tableStorageFormat : getHiveStorageFormat(session);
    // unpartitioned tables ignore the partition storage format
    HiveStorageFormat actualStorageFormat = partitionedBy.isEmpty() ? tableStorageFormat : partitionStorageFormat;
    validateColumns(actualStorageFormat, columnHandles);
    if (tableEncryptionProperties.isPresent() && tableStorageFormat != actualStorageFormat) {
        throw new PrestoException(INVALID_TABLE_PROPERTY, format("For encrypted tables, partition format (%s) should match table format (%s). Using the session property %s or appropriately setting %s can help with ensuring this", actualStorageFormat.name(), tableStorageFormat.name(), RESPECT_TABLE_FORMAT, HIVE_STORAGE_FORMAT));
    }
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Map<String, HiveColumnHandle> columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName);
    List<Column> partitionColumns = partitionedBy.stream().map(columnHandlesByName::get).map(columnHandle -> columnHandleToColumn(metastoreContext, columnHandle)).collect(toList());
    checkPartitionTypesSupported(partitionColumns);
    LocationHandle locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, isTempPathRequired(session, bucketProperty, preferredOrderingColumns));
    HdfsContext context = new HdfsContext(session, schemaName, tableName, locationHandle.getTargetPath().toString(), true);
    Map<String, String> tableProperties = getEmptyTableProperties(tableMetadata, context, tableStorageFormat, tableEncryptionProperties);
    HiveOutputTableHandle result = new HiveOutputTableHandle(schemaName, tableName, columnHandles, metastore.generatePageSinkMetadata(metastoreContext, schemaTableName), locationHandle, tableStorageFormat, partitionStorageFormat, actualStorageFormat, getHiveCompressionCodec(session, false, actualStorageFormat), partitionedBy, bucketProperty, preferredOrderingColumns, session.getUser(), tableProperties, encryptionInformationProvider.getWriteEncryptionInformation(session, tableEncryptionProperties, schemaName, tableName));
    WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
    metastore.declareIntentionToWrite(context, metastoreContext, writeInfo.getWriteMode(), writeInfo.getWritePath(), writeInfo.getTempPath(), schemaTableName, false);
    return result;
}

Example 9 with ConnectorNewTableLayout

use of com.facebook.presto.spi.ConnectorNewTableLayout in project presto by prestodb.

the class RaptorMetadata method getNewTableLayout.

@Override
public Optional<ConnectorNewTableLayout> getNewTableLayout(ConnectorSession session, ConnectorTableMetadata metadata) {
    ImmutableMap.Builder<String, RaptorColumnHandle> map = ImmutableMap.builder();
    long columnId = 1;
    StorageTypeConverter storageTypeConverter = new StorageTypeConverter(typeManager);
    for (ColumnMetadata column : metadata.getColumns()) {
        checkState(storageTypeConverter.toStorageType(column.getType()) != null, "storage type cannot be null");
        map.put(column.getName(), new RaptorColumnHandle(connectorId, column.getName(), columnId, column.getType()));
        columnId++;
    }
    Optional<DistributionInfo> distribution = getOrCreateDistribution(map.build(), metadata.getProperties());
    if (!distribution.isPresent()) {
        return Optional.empty();
    }
    List<String> partitionColumns = distribution.get().getBucketColumns().stream().map(RaptorColumnHandle::getColumnName).collect(toList());
    ConnectorPartitioningHandle partitioning = getPartitioningHandle(distribution.get().getDistributionId());
    return Optional.of(new ConnectorNewTableLayout(partitioning, partitionColumns));
}