Examples with SortingColumn alluxio.grpc.table.layout.hive.SortingColumn used on opensource projects

Example 1 with SortingColumn

use of alluxio.grpc.table.layout.hive.SortingColumn in project alluxio by Alluxio.

the class HiveUtils method toProto.

/**
 * Convert from a StorageDescriptor to a Storage object.
 *
 * @param sd storage descriptor
 * @param translator path translator
 * @return storage proto object
 */
public static Storage toProto(StorageDescriptor sd, PathTranslator translator) throws IOException {
    if (sd == null) {
        return Storage.getDefaultInstance();
    }
    String serDe = sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getSerializationLib();
    Map<String, String> serdeLibMap = sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getParameters();
    StorageFormat.Builder formatBuilder = StorageFormat.newBuilder().setInputFormat(sd.getInputFormat()).setOutputFormat(sd.getOutputFormat());
    if (serdeLibMap != null) {
        formatBuilder.putAllSerdelibParameters(serdeLibMap);
    }
    if (serDe != null) {
        // Check SerDe info
        formatBuilder.setSerde(serDe);
    }
    Storage.Builder storageBuilder = Storage.newBuilder();
    List<Order> orderList = sd.getSortCols();
    List<SortingColumn> sortingColumns;
    if (orderList == null) {
        sortingColumns = Collections.emptyList();
    } else {
        sortingColumns = orderList.stream().map(order -> SortingColumn.newBuilder().setColumnName(order.getCol()).setOrder(order.getOrder() == 1 ? SortingColumn.SortingOrder.ASCENDING : SortingColumn.SortingOrder.DESCENDING).build()).collect(Collectors.toList());
    }
    return storageBuilder.setStorageFormat(formatBuilder.build()).setLocation(translator.toAlluxioPath(sd.getLocation())).setBucketProperty(HiveBucketProperty.newBuilder().setBucketCount(sd.getNumBuckets()).addAllBucketedBy(sd.getBucketCols()).addAllSortedBy(sortingColumns).build()).setSkewed(sd.getSkewedInfo() != null && (sd.getSkewedInfo().getSkewedColNames()) != null && !sd.getSkewedInfo().getSkewedColNames().isEmpty()).putAllSerdeParameters(sd.getParameters()).build();
}

Example 2 with SortingColumn

use of alluxio.grpc.table.layout.hive.SortingColumn in project alluxio by Alluxio.

the class GlueUtils method toProto.

/**
 * Convert the Glue Storage Descriptor and Translator information to Storage.
 *
 * @param sd the glue storage descriptor
 * @param translator the glue translator
 * @return storage proto
 * @throws IOException
 */
public static Storage toProto(StorageDescriptor sd, PathTranslator translator) throws IOException {
    if (sd == null) {
        return Storage.getDefaultInstance();
    }
    String serDe = sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getSerializationLibrary();
    Map<String, String> serdeLibMap = sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getParameters();
    StorageFormat.Builder formatBuilder = StorageFormat.newBuilder().setInputFormat(sd.getInputFormat()).setOutputFormat(sd.getOutputFormat());
    if (serdeLibMap != null) {
        formatBuilder.putAllSerdelibParameters(serdeLibMap);
    }
    if (serDe != null) {
        // Check SerDe info
        formatBuilder.setSerde(serDe);
    }
    alluxio.grpc.table.layout.hive.Storage.Builder storageBuilder = alluxio.grpc.table.layout.hive.Storage.newBuilder();
    List<String> bucketColumn = sd.getBucketColumns() == null ? Collections.emptyList() : sd.getBucketColumns();
    List<Order> orderList = sd.getSortColumns();
    List<SortingColumn> sortingColumns;
    if (orderList == null) {
        sortingColumns = Collections.emptyList();
    } else {
        sortingColumns = orderList.stream().map(order -> SortingColumn.newBuilder().setColumnName(order.getColumn()).setOrder(order.getSortOrder() == 1 ? SortingColumn.SortingOrder.ASCENDING : SortingColumn.SortingOrder.DESCENDING).build()).collect(Collectors.toList());
    }
    return storageBuilder.setStorageFormat(formatBuilder.build()).setLocation(translator.toAlluxioPath(sd.getLocation())).setBucketProperty(HiveBucketProperty.newBuilder().setBucketCount(sd.getNumberOfBuckets()).addAllBucketedBy(bucketColumn).addAllSortedBy(sortingColumns).build()).setSkewed(sd.getSkewedInfo() != null && (sd.getSkewedInfo().getSkewedColumnNames()) != null && !sd.getSkewedInfo().getSkewedColumnNames().isEmpty()).putAllSerdeParameters(sd.getParameters()).build();
}