Examples with TypeManager com.facebook.presto.common.type.TypeManager used on opensource projects

Example 6 with TypeManager

use of com.facebook.presto.common.type.TypeManager in project presto by prestodb.

the class HiveMetadata method getPartitionsSystemTable.

private Optional<SystemTable> getPartitionsSystemTable(ConnectorSession session, SchemaTableName tableName, SchemaTableName sourceTableName) {
    HiveTableHandle sourceTableHandle = getTableHandle(session, sourceTableName);
    if (sourceTableHandle == null) {
        return Optional.empty();
    }
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Table sourceTable = metastore.getTable(metastoreContext, sourceTableName.getSchemaName(), sourceTableName.getTableName()).orElseThrow(() -> new TableNotFoundException(sourceTableName));
    List<HiveColumnHandle> partitionColumns = getPartitionKeyColumnHandles(sourceTable);
    if (partitionColumns.isEmpty()) {
        return Optional.empty();
    }
    List<Type> partitionColumnTypes = partitionColumns.stream().map(HiveColumnHandle::getTypeSignature).map(typeManager::getType).collect(toImmutableList());
    List<ColumnMetadata> partitionSystemTableColumns = partitionColumns.stream().map(column -> new ColumnMetadata(column.getName(), typeManager.getType(column.getTypeSignature()), column.getComment().orElse(null), column.isHidden())).collect(toImmutableList());
    Map<Integer, HiveColumnHandle> fieldIdToColumnHandle = IntStream.range(0, partitionColumns.size()).boxed().collect(toImmutableMap(identity(), partitionColumns::get));
    return Optional.of(createSystemTable(new ConnectorTableMetadata(tableName, partitionSystemTableColumns), constraint -> {
        TupleDomain<ColumnHandle> targetTupleDomain = constraint.transform(fieldIdToColumnHandle::get);
        Predicate<Map<ColumnHandle, NullableValue>> targetPredicate = convertToPredicate(targetTupleDomain);
        Constraint targetConstraint = new Constraint(targetTupleDomain, targetPredicate);
        Iterable<List<Object>> records = () -> stream(partitionManager.getPartitionsIterator(metastore, sourceTableHandle, targetConstraint, session)).map(hivePartition -> IntStream.range(0, partitionColumns.size()).mapToObj(fieldIdToColumnHandle::get).map(columnHandle -> ((HivePartition) hivePartition).getKeys().get(columnHandle).getValue()).collect(toList())).iterator();
        return new InMemoryRecordSet(partitionColumnTypes, records).cursor();
    }));
}

Example 7 with TypeManager

use of com.facebook.presto.common.type.TypeManager in project presto by prestodb.

the class IcebergFileWriterFactory method createOrcWriter.

private IcebergFileWriter createOrcWriter(Path outputPath, Schema icebergSchema, JobConf jobConf, ConnectorSession session) {
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), outputPath, jobConf);
        DataSink orcDataSink = hdfsEnvironment.doAs(session.getUser(), () -> new OutputStreamDataSink(fileSystem.create(outputPath)));
        Callable<Void> rollbackAction = () -> {
            hdfsEnvironment.doAs(session.getUser(), () -> fileSystem.delete(outputPath, false));
            return null;
        };
        List<Types.NestedField> columnFields = icebergSchema.columns();
        List<String> fileColumnNames = columnFields.stream().map(Types.NestedField::name).collect(toImmutableList());
        List<Type> fileColumnTypes = columnFields.stream().map(Types.NestedField::type).map(type -> toPrestoType(type, typeManager)).collect(toImmutableList());
        Optional<Supplier<OrcDataSource>> validationInputFactory = Optional.empty();
        if (isOrcOptimizedWriterValidate(session)) {
            validationInputFactory = Optional.of(() -> {
                try {
                    return new HdfsOrcDataSource(new OrcDataSourceId(outputPath.toString()), hdfsEnvironment.doAs(session.getUser(), () -> fileSystem.getFileStatus(outputPath).getLen()), getOrcMaxMergeDistance(session), getOrcMaxBufferSize(session), getOrcStreamBufferSize(session), false, hdfsEnvironment.doAs(session.getUser(), () -> fileSystem.open(outputPath)), readStats);
                } catch (IOException e) {
                    throw new PrestoException(ICEBERG_WRITE_VALIDATION_FAILED, e);
                }
            });
        }
        return new IcebergOrcFileWriter(icebergSchema, orcDataSink, rollbackAction, ORC, fileColumnNames, fileColumnTypes, toOrcType(icebergSchema), getCompressionCodec(session).getOrcCompressionKind(), orcFileWriterConfig.toOrcWriterOptionsBuilder().withFlushPolicy(DefaultOrcWriterFlushPolicy.builder().withStripeMinSize(HiveSessionProperties.getOrcOptimizedWriterMinStripeSize(session)).withStripeMaxSize(HiveSessionProperties.getOrcOptimizedWriterMaxStripeSize(session)).withStripeMaxRowCount(HiveSessionProperties.getOrcOptimizedWriterMaxStripeRows(session)).build()).withDictionaryMaxMemory(HiveSessionProperties.getOrcOptimizedWriterMaxDictionaryMemory(session)).withMaxStringStatisticsLimit(HiveSessionProperties.getOrcStringStatisticsLimit(session)).build(), IntStream.range(0, fileColumnNames.size()).toArray(), ImmutableMap.<String, String>builder().put(PRESTO_VERSION_NAME, nodeVersion.toString()).put(PRESTO_QUERY_ID_NAME, session.getQueryId()).build(), UTC, validationInputFactory, getOrcOptimizedWriterValidateMode(session), orcWriterStats, dwrfEncryptionProvider, Optional.empty());
    } catch (IOException e) {
        throw new PrestoException(ICEBERG_WRITER_OPEN_ERROR, "Error creating ORC file", e);
    }
}

Example 8 with TypeManager

use of com.facebook.presto.common.type.TypeManager in project presto by prestodb.

the class HiveScalarFunctionInvoker method createFunctionInvoker.

public static HiveScalarFunctionInvoker createFunctionInvoker(Class<?> cls, QualifiedObjectName name, List<TypeSignature> arguments, TypeManager typeManager) {
    final List<Type> argumentTypes = arguments.stream().map(typeManager::getType).collect(Collectors.toList());
    try {
        // Step 1: Create function instance
        final GenericUDF udf = createGenericUDF(name, cls);
        // Step 2: Initialize function
        ObjectInspector[] inputInspectors = argumentTypes.stream().map(argumentType -> ObjectInspectors.create(argumentType, typeManager)).toArray(ObjectInspector[]::new);
        ObjectInspector resultInspector = udf.initialize(inputInspectors);
        // Step 3: Create invoker
        Type resultType = PrestoTypes.fromObjectInspector(resultInspector, typeManager);
        ObjectInputDecoder[] argumentDecoders = argumentTypes.stream().map(argumentsType -> createDecoder(argumentsType, typeManager)).toArray(ObjectInputDecoder[]::new);
        ObjectEncoder resultEncoder = createEncoder(resultType, resultInspector);
        Signature signature = new Signature(name, SCALAR, resultType.getTypeSignature(), arguments);
        // Step 4: Create ThreadLocal GenericUDF
        final ThreadLocal<GenericUDF> genericUDFSupplier = ThreadLocal.withInitial(() -> {
            try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(cls.getClassLoader())) {
                GenericUDF ret = createGenericUDF(name, cls);
                ret.initialize(inputInspectors);
                return ret;
            } catch (Exception e) {
                throw initializationError(e);
            }
        });
        return new HiveScalarFunctionInvoker(signature, genericUDFSupplier::get, argumentDecoders, resultEncoder);
    } catch (Exception e) {
        throw initializationError(e);
    }
}

Example 9 with TypeManager

use of com.facebook.presto.common.type.TypeManager in project presto by prestodb.

the class TestHiveMetadata method testColumnMetadataGetter.

@Test
public void testColumnMetadataGetter() {
    TypeManager mockTypeManager = new TestingTypeManager();
    Column column1 = new Column("c1", HIVE_INT, Optional.empty(), Optional.of("some-metadata"));
    HiveColumnHandle hiveColumnHandle1 = new HiveColumnHandle(column1.getName(), HiveType.HIVE_INT, TypeSignature.parseTypeSignature("int"), 0, HiveColumnHandle.ColumnType.REGULAR, Optional.empty(), Optional.empty());
    HiveColumnHandle hidden = new HiveColumnHandle(HiveColumnHandle.PATH_COLUMN_NAME, HiveType.HIVE_INT, TypeSignature.parseTypeSignature("int"), 0, HiveColumnHandle.ColumnType.SYNTHESIZED, Optional.empty(), Optional.empty());
    Column partitionColumn = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty());
    Table mockTable = new Table("schema", "table", "user", PrestoTableType.MANAGED_TABLE, new Storage(fromHiveStorageFormat(ORC), "location", Optional.of(new HiveBucketProperty(ImmutableList.of(column1.getName()), 100, ImmutableList.of(), HIVE_COMPATIBLE, Optional.empty())), false, ImmutableMap.of(), ImmutableMap.of()), ImmutableList.of(column1), ImmutableList.of(partitionColumn), ImmutableMap.of(), Optional.empty(), Optional.empty());
    ColumnMetadata actual = HiveMetadata.columnMetadataGetter(mockTable, mockTypeManager, new HiveColumnConverter()).apply(hiveColumnHandle1);
    ColumnMetadata expected = new ColumnMetadata("c1", IntegerType.INTEGER);
    assertEquals(actual, expected);
    actual = HiveMetadata.columnMetadataGetter(mockTable, mockTypeManager, new TestColumnConverter()).apply(hidden);
    expected = ColumnMetadata.builder().setName(HiveColumnHandle.PATH_COLUMN_NAME).setType(IntegerType.INTEGER).setHidden(true).build();
    assertEquals(actual, expected);
}

Example 10 with TypeManager

use of com.facebook.presto.common.type.TypeManager in project presto by prestodb.

the class TestIonSqlQueryBuilder method testDecimalColumns.

@Test
public void testDecimalColumns() {
    TypeManager typeManager = createTestFunctionAndTypeManager();
    IonSqlQueryBuilder queryBuilder = new IonSqlQueryBuilder(typeManager);
    List<HiveColumnHandle> columns = ImmutableList.of(new HiveColumnHandle("quantity", HiveType.valueOf("decimal(20,0)"), parseTypeSignature(DECIMAL), 0, REGULAR, Optional.empty(), Optional.empty()), new HiveColumnHandle("extendedprice", HiveType.valueOf("decimal(20,2)"), parseTypeSignature(DECIMAL), 1, REGULAR, Optional.empty(), Optional.empty()), new HiveColumnHandle("discount", HiveType.valueOf("decimal(10,2)"), parseTypeSignature(DECIMAL), 2, REGULAR, Optional.empty(), Optional.empty()));
    DecimalType decimalType = DecimalType.createDecimalType(10, 2);
    TupleDomain<HiveColumnHandle> tupleDomain = withColumnDomains(ImmutableMap.of(columns.get(0), Domain.create(ofRanges(Range.lessThan(DecimalType.createDecimalType(20, 0), longDecimal("50"))), false), columns.get(1), Domain.create(ofRanges(Range.equal(HiveType.valueOf("decimal(20,2)").getType(typeManager), longDecimal("0.05"))), false), columns.get(2), Domain.create(ofRanges(Range.range(decimalType, shortDecimal("0.0"), true, shortDecimal("0.02"), true)), false)));
    assertEquals("SELECT s._1, s._2, s._3 FROM S3Object s WHERE ((case s._1 when '' then null else CAST(s._1 AS DECIMAL(20,0)) end < 50)) AND " + "(case s._2 when '' then null else CAST(s._2 AS DECIMAL(20,2)) end = 0.05) AND ((case s._3 when '' then null else CAST(s._3 AS DECIMAL(10,2)) " + "end >= 0.00 AND case s._3 when '' then null else CAST(s._3 AS DECIMAL(10,2)) end <= 0.02))", queryBuilder.buildSql(columns, tupleDomain));
}