Examples with HiveBucketProperty com.facebook.presto.hive.HiveBucketProperty used on opensource projects

Example 1 with HiveBucketProperty

use of com.facebook.presto.hive.HiveBucketProperty in project presto by prestodb.

the class TestGlueToPrestoConverter method assertStorage.

private static void assertStorage(Storage actual, StorageDescriptor expected) {
    assertEquals(actual.getLocation(), expected.getLocation());
    assertEquals(actual.getStorageFormat().getSerDe(), expected.getSerdeInfo().getSerializationLibrary());
    assertEquals(actual.getStorageFormat().getInputFormat(), expected.getInputFormat());
    assertEquals(actual.getStorageFormat().getOutputFormat(), expected.getOutputFormat());
    if (!isNullOrEmpty(expected.getBucketColumns())) {
        HiveBucketProperty bucketProperty = actual.getBucketProperty().get();
        assertEquals(bucketProperty.getBucketedBy(), expected.getBucketColumns());
        assertEquals(bucketProperty.getBucketCount(), expected.getNumberOfBuckets().intValue());
    }
}

Example 2 with HiveBucketProperty

use of com.facebook.presto.hive.HiveBucketProperty in project presto by prestodb.

the class GlueInputConverter method convertStorage.

private static StorageDescriptor convertStorage(Storage storage, List<Column> columns) {
    if (storage.isSkewed()) {
        throw new IllegalArgumentException("Writing to skewed table/partition is not supported");
    }
    SerDeInfo serdeInfo = new SerDeInfo().withSerializationLibrary(storage.getStorageFormat().getSerDeNullable()).withParameters(storage.getSerdeParameters());
    StorageDescriptor sd = new StorageDescriptor();
    sd.setLocation(storage.getLocation());
    sd.setColumns(columns.stream().map(GlueInputConverter::convertColumn).collect(toList()));
    sd.setSerdeInfo(serdeInfo);
    sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable());
    sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable());
    sd.setParameters(ImmutableMap.of());
    Optional<HiveBucketProperty> bucketProperty = storage.getBucketProperty();
    if (bucketProperty.isPresent()) {
        sd.setNumberOfBuckets(bucketProperty.get().getBucketCount());
        sd.setBucketColumns(bucketProperty.get().getBucketedBy());
        if (!bucketProperty.get().getSortedBy().isEmpty()) {
            sd.setSortColumns(bucketProperty.get().getSortedBy().stream().map(column -> new Order().withColumn(column.getColumnName()).withSortOrder(column.getOrder().getHiveOrder())).collect(toImmutableList()));
        }
    }
    return sd;
}

Example 3 with HiveBucketProperty

use of com.facebook.presto.hive.HiveBucketProperty in project presto by prestodb.

the class ThriftMetastoreUtil method makeStorageDescriptor.

private static StorageDescriptor makeStorageDescriptor(String tableName, List<Column> columns, Storage storage, ColumnConverter columnConverter) {
    if (storage.isSkewed()) {
        throw new IllegalArgumentException("Writing to skewed table/partition is not supported");
    }
    SerDeInfo serdeInfo = new SerDeInfo();
    serdeInfo.setName(tableName);
    serdeInfo.setSerializationLib(storage.getStorageFormat().getSerDeNullable());
    serdeInfo.setParameters(storage.getSerdeParameters());
    StorageDescriptor sd = new StorageDescriptor();
    sd.setLocation(emptyToNull(storage.getLocation()));
    sd.setCols(columns.stream().map(col -> ThriftMetastoreUtil.toMetastoreApiFieldSchema(col, columnConverter)).collect(toList()));
    sd.setSerdeInfo(serdeInfo);
    sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable());
    sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable());
    sd.setParameters(storage.getParameters());
    Optional<HiveBucketProperty> bucketProperty = storage.getBucketProperty();
    if (bucketProperty.isPresent()) {
        sd.setNumBuckets(bucketProperty.get().getBucketCount());
        sd.setBucketCols(bucketProperty.get().getBucketedBy());
        if (!bucketProperty.get().getSortedBy().isEmpty()) {
            sd.setSortCols(bucketProperty.get().getSortedBy().stream().map(column -> new Order(column.getColumnName(), column.getOrder().getHiveOrder())).collect(toList()));
        }
    }
    return sd;
}

Example 4 with HiveBucketProperty

use of com.facebook.presto.hive.HiveBucketProperty in project presto by prestodb.

the class MetastoreUtil method makeStorageDescriptor.

private static StorageDescriptor makeStorageDescriptor(String tableName, List<Column> columns, Storage storage) {
    if (storage.isSorted() || storage.isSkewed()) {
        throw new IllegalArgumentException("Writing to sorted and/or skewed table/partition is not supported");
    }
    SerDeInfo serdeInfo = new SerDeInfo();
    serdeInfo.setName(tableName);
    serdeInfo.setSerializationLib(storage.getStorageFormat().getSerDeNullable());
    serdeInfo.setParameters(storage.getSerdeParameters());
    StorageDescriptor sd = new StorageDescriptor();
    sd.setLocation(emptyToNull(storage.getLocation()));
    sd.setCols(columns.stream().map(MetastoreUtil::toMetastoreApiFieldSchema).collect(toList()));
    sd.setSerdeInfo(serdeInfo);
    sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable());
    sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable());
    sd.setParameters(ImmutableMap.of());
    Optional<HiveBucketProperty> bucketProperty = storage.getBucketProperty();
    if (bucketProperty.isPresent()) {
        sd.setNumBuckets(bucketProperty.get().getBucketCount());
        sd.setBucketCols(bucketProperty.get().getBucketedBy());
    }
    return sd;
}

Example 5 with HiveBucketProperty

use of com.facebook.presto.hive.HiveBucketProperty in project presto by prestodb.

the class TestGlueInputConverter method assertStorage.

private static void assertStorage(StorageDescriptor actual, Storage expected) {
    assertEquals(actual.getLocation(), expected.getLocation());
    assertEquals(actual.getSerdeInfo().getSerializationLibrary(), expected.getStorageFormat().getSerDe());
    assertEquals(actual.getInputFormat(), expected.getStorageFormat().getInputFormat());
    assertEquals(actual.getOutputFormat(), expected.getStorageFormat().getOutputFormat());
    if (expected.getBucketProperty().isPresent()) {
        HiveBucketProperty bucketProperty = expected.getBucketProperty().get();
        assertEquals(actual.getBucketColumns(), bucketProperty.getBucketedBy());
        assertEquals(actual.getNumberOfBuckets().intValue(), bucketProperty.getBucketCount());
    }
}