use of com.facebook.presto.common.type.TypeSignature in project presto by prestodb.
the class UserDefinedTypeRowMapper method map.
@Override
public UserDefinedType map(ResultSet rs, StatementContext ctx) throws SQLException {
QualifiedObjectName typeName = QualifiedObjectName.valueOf(rs.getString("catalog_name"), rs.getString("schema_name"), rs.getString("type_name"));
String physicalType = rs.getString("physical_type");
TypeSignature typeSignature = parseTypeSignature(physicalType);
return new UserDefinedType(typeName, typeSignature);
}
use of com.facebook.presto.common.type.TypeSignature in project presto by prestodb.
the class JsonUtil method canCastFromJson.
public static boolean canCastFromJson(Type type) {
TypeSignature signature = type.getTypeSignature();
String baseType = signature.getBase();
if (signature.isEnum()) {
return true;
}
if (baseType.equals(StandardTypes.BOOLEAN) || baseType.equals(StandardTypes.TINYINT) || baseType.equals(StandardTypes.SMALLINT) || baseType.equals(StandardTypes.INTEGER) || baseType.equals(StandardTypes.BIGINT) || baseType.equals(StandardTypes.REAL) || baseType.equals(StandardTypes.DOUBLE) || baseType.equals(StandardTypes.VARCHAR) || baseType.equals(StandardTypes.DECIMAL) || baseType.equals(StandardTypes.JSON)) {
return true;
}
if (type instanceof ArrayType) {
return canCastFromJson(((ArrayType) type).getElementType());
}
if (type instanceof MapType) {
return isValidJsonObjectKeyType(((MapType) type).getKeyType()) && canCastFromJson(((MapType) type).getValueType());
}
if (type instanceof RowType) {
return type.getTypeParameters().stream().allMatch(JsonUtil::canCastFromJson);
}
return false;
}
use of com.facebook.presto.common.type.TypeSignature in project presto by prestodb.
the class OrcStorageManager method toOrcFileType.
static Type toOrcFileType(Type raptorType, TypeManager typeManager) {
// TIMESTAMPS are stored as BIGINT to void the poor encoding in ORC
if (raptorType == TimestampType.TIMESTAMP) {
return BIGINT;
}
if (raptorType instanceof ArrayType) {
Type elementType = toOrcFileType(((ArrayType) raptorType).getElementType(), typeManager);
return new ArrayType(elementType);
}
if (raptorType instanceof MapType) {
TypeSignature keyType = toOrcFileType(((MapType) raptorType).getKeyType(), typeManager).getTypeSignature();
TypeSignature valueType = toOrcFileType(((MapType) raptorType).getValueType(), typeManager).getTypeSignature();
return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.of(keyType), TypeSignatureParameter.of(valueType)));
}
if (raptorType instanceof RowType) {
List<RowType.Field> fields = ((RowType) raptorType).getFields().stream().map(field -> new RowType.Field(field.getName(), toOrcFileType(field.getType(), typeManager))).collect(toImmutableList());
return RowType.from(fields);
}
return raptorType;
}
use of com.facebook.presto.common.type.TypeSignature in project presto by prestodb.
the class OrcFileRewriter method rewrite.
public OrcFileInfo rewrite(FileSystem fileSystem, Map<String, Type> allColumnTypes, Path input, Path output, BitSet rowsToDelete) throws IOException {
try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(FileSystem.class.getClassLoader());
OrcDataSource dataSource = orcDataEnvironment.createOrcDataSource(fileSystem, input, readerAttributes)) {
OrcReader reader = new OrcReader(dataSource, ORC, orcFileTailSource, stripeMetadataSourceFactory, new RaptorOrcAggregatedMemoryContext(), new OrcReaderOptions(readerAttributes.getMaxMergeDistance(), readerAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE, readerAttributes.isZstdJniDecompressionEnabled()), false, NO_ENCRYPTION, DwrfKeyProvider.EMPTY, new RuntimeStats());
if (reader.getFooter().getNumberOfRows() < rowsToDelete.length()) {
throw new IOException("File has fewer rows than deletion vector");
}
int deleteRowCount = rowsToDelete.cardinality();
if (reader.getFooter().getNumberOfRows() == deleteRowCount) {
return new OrcFileInfo(0, 0);
}
if (reader.getFooter().getNumberOfRows() >= Integer.MAX_VALUE) {
throw new IOException("File has too many rows");
}
int inputRowCount = toIntExact(reader.getFooter().getNumberOfRows());
Map<String, Integer> currentColumnIds = IntStream.range(0, reader.getColumnNames().size()).boxed().collect(toMap(reader.getColumnNames()::get, i -> i));
ImmutableList.Builder<Type> writerColumnTypesBuilder = ImmutableList.builder();
ImmutableList.Builder<String> writerColumnIdsBuilder = ImmutableList.builder();
ImmutableList.Builder<Integer> readerColumnIndexBuilder = ImmutableList.builder();
// Build columns for writer; keep the right ordinal
Map<String, Type> orderedAllColumnTypes = new TreeMap<>(Comparator.comparingLong(Long::parseLong));
orderedAllColumnTypes.putAll(allColumnTypes);
for (Map.Entry<String, Type> columnType : orderedAllColumnTypes.entrySet()) {
// Get the intersection of the provide columns and the actual columns
Integer currentColumnIndex = currentColumnIds.get(columnType.getKey());
if (currentColumnIndex != null) {
readerColumnIndexBuilder.add(currentColumnIndex);
writerColumnTypesBuilder.add(columnType.getValue());
writerColumnIdsBuilder.add(columnType.getKey());
}
}
List<Type> writerColumnTypes = writerColumnTypesBuilder.build();
List<String> writerColumnIds = writerColumnIdsBuilder.build();
List<Integer> readerColumnIndex = readerColumnIndexBuilder.build();
Map<Integer, Type> readerColumns = IntStream.range(0, readerColumnIndex.size()).boxed().collect(toMap(readerColumnIndex::get, writerColumnTypes::get));
if (writerColumnTypes.isEmpty()) {
// no intersection; directly return
return new OrcFileInfo(0, 0);
}
StorageTypeConverter converter = new StorageTypeConverter(typeManager);
List<Type> writerStorageTypes = writerColumnTypes.stream().map(converter::toStorageType).collect(toImmutableList());
long start = System.nanoTime();
Map<String, String> userMetadata = ImmutableMap.of();
if (reader.getFooter().getUserMetadata().containsKey(OrcFileMetadata.KEY)) {
// build metadata if the original file has it
ImmutableMap.Builder<Long, TypeSignature> metadataBuilder = ImmutableMap.builder();
for (int i = 0; i < writerColumnIds.size(); i++) {
metadataBuilder.put(Long.parseLong(writerColumnIds.get(i)), writerColumnTypes.get(i).getTypeSignature());
}
userMetadata = ImmutableMap.of(OrcFileMetadata.KEY, METADATA_CODEC.toJson(new OrcFileMetadata(metadataBuilder.build())));
}
StorageTypeConverter storageTypeConverter = new StorageTypeConverter(typeManager);
try (Closer<OrcBatchRecordReader, IOException> recordReader = closer(reader.createBatchRecordReader(storageTypeConverter.toStorageTypes(readerColumns), TRUE, DEFAULT_STORAGE_TIMEZONE, new RaptorOrcAggregatedMemoryContext(), INITIAL_BATCH_SIZE), OrcBatchRecordReader::close);
Closer<OrcWriter, IOException> writer = closer(new OrcWriter(orcDataEnvironment.createOrcDataSink(fileSystem, output), writerColumnIds, writerStorageTypes, ORC, compression, Optional.empty(), NO_ENCRYPTION, getDefaultOrcWriterOptions(), userMetadata, DEFAULT_STORAGE_TIMEZONE, validate, HASHED, stats), OrcWriter::close)) {
OrcFileInfo fileInfo = rewrite(recordReader.get(), writer.get(), rowsToDelete, writerColumnTypes, readerColumnIndexBuilder.build());
log.debug("Rewrote file %s in %s (input rows: %s, output rows: %s)", input.getName(), nanosSince(start), inputRowCount, inputRowCount - deleteRowCount);
return fileInfo;
}
} catch (NotSupportedException e) {
throw new PrestoException(NOT_SUPPORTED, e.getMessage(), e);
}
}
use of com.facebook.presto.common.type.TypeSignature in project presto by prestodb.
the class QueryRewriter method getColumnTypeRewrite.
private Optional<Type> getColumnTypeRewrite(Type type) {
if (type.equals(DATE) || type.equals(TIME)) {
return Optional.of(TIMESTAMP);
}
if (type.equals(TIMESTAMP_WITH_TIME_ZONE)) {
return Optional.of(VARCHAR);
}
if (type.equals(UNKNOWN)) {
return Optional.of(BIGINT);
}
if (type instanceof DecimalType) {
return Optional.of(DOUBLE);
}
if (type instanceof ArrayType) {
return getColumnTypeRewrite(((ArrayType) type).getElementType()).map(ArrayType::new);
}
if (type instanceof MapType) {
Type keyType = ((MapType) type).getKeyType();
Type valueType = ((MapType) type).getValueType();
Optional<Type> keyTypeRewrite = getColumnTypeRewrite(keyType);
Optional<Type> valueTypeRewrite = getColumnTypeRewrite(valueType);
if (keyTypeRewrite.isPresent() || valueTypeRewrite.isPresent()) {
return Optional.of(typeManager.getType(new TypeSignature(MAP, TypeSignatureParameter.of(keyTypeRewrite.orElse(keyType).getTypeSignature()), TypeSignatureParameter.of(valueTypeRewrite.orElse(valueType).getTypeSignature()))));
}
return Optional.empty();
}
if (type instanceof RowType) {
List<Field> fields = ((RowType) type).getFields();
List<Field> fieldsRewrite = new ArrayList<>();
boolean rewrite = false;
for (Field field : fields) {
Optional<Type> fieldTypeRewrite = getColumnTypeRewrite(field.getType());
rewrite = rewrite || fieldTypeRewrite.isPresent();
fieldsRewrite.add(new Field(field.getName(), fieldTypeRewrite.orElse(field.getType())));
}
return rewrite ? Optional.of(RowType.from(fieldsRewrite)) : Optional.empty();
}
return Optional.empty();
}
Aggregations