use of alluxio.grpc.table.layout.hive.Storage in project alluxio by Alluxio.
the class HiveUtils method toProto.
/**
* Convert from a StorageDescriptor to a Storage object.
*
* @param sd storage descriptor
* @param translator path translator
* @return storage proto object
*/
public static Storage toProto(StorageDescriptor sd, PathTranslator translator) throws IOException {
if (sd == null) {
return Storage.getDefaultInstance();
}
String serDe = sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getSerializationLib();
Map<String, String> serdeLibMap = sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getParameters();
StorageFormat.Builder formatBuilder = StorageFormat.newBuilder().setInputFormat(sd.getInputFormat()).setOutputFormat(sd.getOutputFormat());
if (serdeLibMap != null) {
formatBuilder.putAllSerdelibParameters(serdeLibMap);
}
if (serDe != null) {
// Check SerDe info
formatBuilder.setSerde(serDe);
}
Storage.Builder storageBuilder = Storage.newBuilder();
List<Order> orderList = sd.getSortCols();
List<SortingColumn> sortingColumns;
if (orderList == null) {
sortingColumns = Collections.emptyList();
} else {
sortingColumns = orderList.stream().map(order -> SortingColumn.newBuilder().setColumnName(order.getCol()).setOrder(order.getOrder() == 1 ? SortingColumn.SortingOrder.ASCENDING : SortingColumn.SortingOrder.DESCENDING).build()).collect(Collectors.toList());
}
return storageBuilder.setStorageFormat(formatBuilder.build()).setLocation(translator.toAlluxioPath(sd.getLocation())).setBucketProperty(HiveBucketProperty.newBuilder().setBucketCount(sd.getNumBuckets()).addAllBucketedBy(sd.getBucketCols()).addAllSortedBy(sortingColumns).build()).setSkewed(sd.getSkewedInfo() != null && (sd.getSkewedInfo().getSkewedColNames()) != null && !sd.getSkewedInfo().getSkewedColNames().isEmpty()).putAllSerdeParameters(sd.getParameters()).build();
}
use of alluxio.grpc.table.layout.hive.Storage in project alluxio by Alluxio.
the class GlueUtils method toProto.
/**
* Convert the Glue Storage Descriptor and Translator information to Storage.
*
* @param sd the glue storage descriptor
* @param translator the glue translator
* @return storage proto
* @throws IOException
*/
public static Storage toProto(StorageDescriptor sd, PathTranslator translator) throws IOException {
if (sd == null) {
return Storage.getDefaultInstance();
}
String serDe = sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getSerializationLibrary();
Map<String, String> serdeLibMap = sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getParameters();
StorageFormat.Builder formatBuilder = StorageFormat.newBuilder().setInputFormat(sd.getInputFormat()).setOutputFormat(sd.getOutputFormat());
if (serdeLibMap != null) {
formatBuilder.putAllSerdelibParameters(serdeLibMap);
}
if (serDe != null) {
// Check SerDe info
formatBuilder.setSerde(serDe);
}
alluxio.grpc.table.layout.hive.Storage.Builder storageBuilder = alluxio.grpc.table.layout.hive.Storage.newBuilder();
List<String> bucketColumn = sd.getBucketColumns() == null ? Collections.emptyList() : sd.getBucketColumns();
List<Order> orderList = sd.getSortColumns();
List<SortingColumn> sortingColumns;
if (orderList == null) {
sortingColumns = Collections.emptyList();
} else {
sortingColumns = orderList.stream().map(order -> SortingColumn.newBuilder().setColumnName(order.getColumn()).setOrder(order.getSortOrder() == 1 ? SortingColumn.SortingOrder.ASCENDING : SortingColumn.SortingOrder.DESCENDING).build()).collect(Collectors.toList());
}
return storageBuilder.setStorageFormat(formatBuilder.build()).setLocation(translator.toAlluxioPath(sd.getLocation())).setBucketProperty(HiveBucketProperty.newBuilder().setBucketCount(sd.getNumberOfBuckets()).addAllBucketedBy(bucketColumn).addAllSortedBy(sortingColumns).build()).setSkewed(sd.getSkewedInfo() != null && (sd.getSkewedInfo().getSkewedColumnNames()) != null && !sd.getSkewedInfo().getSkewedColumnNames().isEmpty()).putAllSerdeParameters(sd.getParameters()).build();
}
Aggregations