Search in sources :

Example 21 with ColumnHandle

use of com.facebook.presto.spi.ColumnHandle in project presto by prestodb.

the class HivePartitionManager method parseValuesAndFilterPartition.

private Optional<Map<ColumnHandle, NullableValue>> parseValuesAndFilterPartition(String partitionName, List<HiveColumnHandle> partitionColumns, List<Type> partitionTypes, Constraint<ColumnHandle> constraint) {
    List<String> partitionValues = extractPartitionKeyValues(partitionName);
    Map<ColumnHandle, Domain> domains = constraint.getSummary().getDomains().get();
    ImmutableMap.Builder<ColumnHandle, NullableValue> builder = ImmutableMap.builder();
    for (int i = 0; i < partitionColumns.size(); i++) {
        HiveColumnHandle column = partitionColumns.get(i);
        NullableValue parsedValue = parsePartitionValue(partitionName, partitionValues.get(i), partitionTypes.get(i), timeZone);
        Domain allowedDomain = domains.get(column);
        if (allowedDomain != null && !allowedDomain.includesNullableValue(parsedValue.getValue())) {
            return Optional.empty();
        }
        builder.put(column, parsedValue);
    }
    Map<ColumnHandle, NullableValue> values = builder.build();
    if (!constraint.predicate().test(values)) {
        return Optional.empty();
    }
    return Optional.of(values);
}
Also used : ColumnHandle(com.facebook.presto.spi.ColumnHandle) NullableValue(com.facebook.presto.spi.predicate.NullableValue) ProtectMode.getProtectModeFromString(org.apache.hadoop.hive.metastore.ProtectMode.getProtectModeFromString) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) Domain(com.facebook.presto.spi.predicate.Domain) ImmutableMap(com.google.common.collect.ImmutableMap) Constraint(com.facebook.presto.spi.Constraint)

Example 22 with ColumnHandle

use of com.facebook.presto.spi.ColumnHandle in project presto by prestodb.

the class HivePartitionManager method toCompactTupleDomain.

private static TupleDomain<HiveColumnHandle> toCompactTupleDomain(TupleDomain<ColumnHandle> effectivePredicate, int threshold) {
    checkArgument(effectivePredicate.getDomains().isPresent());
    ImmutableMap.Builder<HiveColumnHandle, Domain> builder = ImmutableMap.builder();
    for (Map.Entry<ColumnHandle, Domain> entry : effectivePredicate.getDomains().get().entrySet()) {
        HiveColumnHandle hiveColumnHandle = (HiveColumnHandle) entry.getKey();
        ValueSet values = entry.getValue().getValues();
        ValueSet compactValueSet = values.getValuesProcessor().<Optional<ValueSet>>transform(ranges -> ranges.getRangeCount() > threshold ? Optional.of(ValueSet.ofRanges(ranges.getSpan())) : Optional.empty(), discreteValues -> discreteValues.getValues().size() > threshold ? Optional.of(ValueSet.all(values.getType())) : Optional.empty(), allOrNone -> Optional.empty()).orElse(values);
        builder.put(hiveColumnHandle, Domain.create(compactValueSet, entry.getValue().isNullAllowed()));
    }
    return TupleDomain.withColumnDomains(builder.build());
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) Table(com.facebook.presto.hive.metastore.Table) TypeManager(com.facebook.presto.spi.type.TypeManager) Slice(io.airlift.slice.Slice) HiveUtil.getPartitionKeyColumnHandles(com.facebook.presto.hive.HiveUtil.getPartitionKeyColumnHandles) Strings.isNullOrEmpty(com.google.common.base.Strings.isNullOrEmpty) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) ProtectMode(org.apache.hadoop.hive.metastore.ProtectMode) PrestoException(com.facebook.presto.spi.PrestoException) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ImmutableList(com.google.common.collect.ImmutableList) Predicates.not(com.google.common.base.Predicates.not) ValueSet(com.facebook.presto.spi.predicate.ValueSet) Type(com.facebook.presto.spi.type.Type) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) Predicates(com.google.common.base.Predicates) HiveBucket(com.facebook.presto.hive.HiveBucketing.HiveBucket) NullableValue(com.facebook.presto.spi.predicate.NullableValue) HIVE_EXCEEDED_PARTITION_LIMIT(com.facebook.presto.hive.HiveErrorCode.HIVE_EXCEEDED_PARTITION_LIMIT) ImmutableMap(com.google.common.collect.ImmutableMap) ProtectMode.getProtectModeFromString(org.apache.hadoop.hive.metastore.ProtectMode.getProtectModeFromString) Constraint(com.facebook.presto.spi.Constraint) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) HiveBucketing.getHiveBucketHandle(com.facebook.presto.hive.HiveBucketing.getHiveBucketHandle) Maps(com.google.common.collect.Maps) String.format(java.lang.String.format) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) Domain(com.facebook.presto.spi.predicate.Domain) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) ColumnHandle(com.facebook.presto.spi.ColumnHandle) HiveBucketing.getHiveBucketNumbers(com.facebook.presto.hive.HiveBucketing.getHiveBucketNumbers) FileUtils(org.apache.hadoop.hive.common.FileUtils) Optional(java.util.Optional) HiveUtil.parsePartitionValue(com.facebook.presto.hive.HiveUtil.parsePartitionValue) ColumnHandle(com.facebook.presto.spi.ColumnHandle) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) Domain(com.facebook.presto.spi.predicate.Domain) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) ValueSet(com.facebook.presto.spi.predicate.ValueSet) ImmutableMap(com.google.common.collect.ImmutableMap)

Example 23 with ColumnHandle

use of com.facebook.presto.spi.ColumnHandle in project presto by prestodb.

the class HiveBucketing method getHiveBucket.

private static Optional<HiveBucket> getHiveBucket(Table table, Map<ColumnHandle, NullableValue> bindings) {
    if (bindings.isEmpty()) {
        return Optional.empty();
    }
    List<String> bucketColumns = table.getStorage().getBucketProperty().get().getBucketedBy();
    Map<String, ObjectInspector> objectInspectors = new HashMap<>();
    // Get column name to object inspector mapping
    for (StructField field : getTableStructFields(table)) {
        objectInspectors.put(field.getFieldName(), field.getFieldObjectInspector());
    }
    // Verify the bucket column types are supported
    for (String column : bucketColumns) {
        ObjectInspector inspector = objectInspectors.get(column);
        if ((inspector == null) || (inspector.getCategory() != Category.PRIMITIVE)) {
            return Optional.empty();
        }
        if (!SUPPORTED_TYPES.contains(((PrimitiveObjectInspector) inspector).getPrimitiveCategory())) {
            return Optional.empty();
        }
    }
    // Get bindings for bucket columns
    Map<String, Object> bucketBindings = new HashMap<>();
    for (Entry<ColumnHandle, NullableValue> entry : bindings.entrySet()) {
        HiveColumnHandle colHandle = (HiveColumnHandle) entry.getKey();
        if (!entry.getValue().isNull() && bucketColumns.contains(colHandle.getName())) {
            bucketBindings.put(colHandle.getName(), entry.getValue().getValue());
        }
    }
    // Check that we have bindings for all bucket columns
    if (bucketBindings.size() != bucketColumns.size()) {
        return Optional.empty();
    }
    // Get bindings of bucket columns
    ImmutableList.Builder<Entry<ObjectInspector, Object>> columnBindings = ImmutableList.builder();
    for (String column : bucketColumns) {
        columnBindings.add(immutableEntry(objectInspectors.get(column), bucketBindings.get(column)));
    }
    return getHiveBucket(columnBindings.build(), table.getStorage().getBucketProperty().get().getBucketCount());
}
Also used : ColumnHandle(com.facebook.presto.spi.ColumnHandle) PrimitiveObjectInspectorFactory.javaByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaByteObjectInspector) PrimitiveObjectInspectorFactory.javaLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaLongObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) PrimitiveObjectInspectorFactory.javaIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaIntObjectInspector) PrimitiveObjectInspectorFactory.javaBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaBooleanObjectInspector) IntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector) PrimitiveObjectInspectorFactory.javaShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaShortObjectInspector) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) PrimitiveObjectInspectorFactory.javaStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector) HashMap(java.util.HashMap) ImmutableList(com.google.common.collect.ImmutableList) NullableValue(com.facebook.presto.spi.predicate.NullableValue) Maps.immutableEntry(com.google.common.collect.Maps.immutableEntry) Entry(java.util.Map.Entry) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) DeferredJavaObject(org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject) DeferredObject(org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject)

Example 24 with ColumnHandle

use of com.facebook.presto.spi.ColumnHandle in project presto by prestodb.

the class RenameColumnTask method execute.

@Override
public ListenableFuture<?> execute(RenameColumn statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List<Expression> parameters) {
    Session session = stateMachine.getSession();
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTable());
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
    String source = statement.getSource().toLowerCase(ENGLISH);
    String target = statement.getTarget().toLowerCase(ENGLISH);
    if (!tableHandle.isPresent()) {
        throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
    }
    accessControl.checkCanRenameColumn(session.getRequiredTransactionId(), session.getIdentity(), tableName);
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle.get());
    if (!columnHandles.containsKey(source)) {
        throw new SemanticException(MISSING_COLUMN, statement, "Column '%s' does not exist", source);
    }
    if (columnHandles.containsKey(target)) {
        throw new SemanticException(COLUMN_ALREADY_EXISTS, statement, "Column '%s' already exists", target);
    }
    metadata.renameColumn(session, tableHandle.get(), columnHandles.get(source), target);
    return immediateFuture(null);
}
Also used : ColumnHandle(com.facebook.presto.spi.ColumnHandle) TableHandle(com.facebook.presto.metadata.TableHandle) QualifiedObjectName(com.facebook.presto.metadata.QualifiedObjectName) MetadataUtil.createQualifiedObjectName(com.facebook.presto.metadata.MetadataUtil.createQualifiedObjectName) Session(com.facebook.presto.Session) SemanticException(com.facebook.presto.sql.analyzer.SemanticException)

Example 25 with ColumnHandle

use of com.facebook.presto.spi.ColumnHandle in project presto by prestodb.

the class InformationSchemaSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    InformationSchemaTableLayoutHandle handle = (InformationSchemaTableLayoutHandle) layout;
    Map<ColumnHandle, NullableValue> bindings = extractFixedValues(handle.getConstraint()).orElse(ImmutableMap.of());
    List<HostAddress> localAddress = ImmutableList.of(nodeManager.getCurrentNode().getHostAndPort());
    Map<String, NullableValue> filters = bindings.entrySet().stream().collect(toMap(entry -> ((InformationSchemaColumnHandle) entry.getKey()).getColumnName(), Entry::getValue));
    ConnectorSplit split = new InformationSchemaSplit(handle.getTable(), filters, localAddress);
    return new FixedSplitSource(ImmutableList.of(split));
}
Also used : ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) ImmutableMap(com.google.common.collect.ImmutableMap) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) HostAddress(com.facebook.presto.spi.HostAddress) FixedSplitSource(com.facebook.presto.spi.FixedSplitSource) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) List(java.util.List) InternalNodeManager(com.facebook.presto.metadata.InternalNodeManager) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toMap(java.util.stream.Collectors.toMap) ColumnHandle(com.facebook.presto.spi.ColumnHandle) TupleDomain.extractFixedValues(com.facebook.presto.spi.predicate.TupleDomain.extractFixedValues) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) Entry(java.util.Map.Entry) NullableValue(com.facebook.presto.spi.predicate.NullableValue) ColumnHandle(com.facebook.presto.spi.ColumnHandle) NullableValue(com.facebook.presto.spi.predicate.NullableValue) HostAddress(com.facebook.presto.spi.HostAddress) FixedSplitSource(com.facebook.presto.spi.FixedSplitSource) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit)

Aggregations

ColumnHandle (com.facebook.presto.spi.ColumnHandle)243 ImmutableList (com.google.common.collect.ImmutableList)90 ImmutableMap (com.google.common.collect.ImmutableMap)81 Test (org.testng.annotations.Test)81 ConnectorSession (com.facebook.presto.spi.ConnectorSession)80 ConnectorTableHandle (com.facebook.presto.spi.ConnectorTableHandle)70 Constraint (com.facebook.presto.spi.Constraint)66 Map (java.util.Map)65 SchemaTableName (com.facebook.presto.spi.SchemaTableName)60 Optional (java.util.Optional)57 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)54 List (java.util.List)54 Objects.requireNonNull (java.util.Objects.requireNonNull)53 ColumnMetadata (com.facebook.presto.spi.ColumnMetadata)47 ConnectorMetadata (com.facebook.presto.spi.connector.ConnectorMetadata)47 ConnectorTableLayoutHandle (com.facebook.presto.spi.ConnectorTableLayoutHandle)46 ImmutableSet (com.google.common.collect.ImmutableSet)46 TestingConnectorSession (com.facebook.presto.testing.TestingConnectorSession)43 ConnectorTableMetadata (com.facebook.presto.spi.ConnectorTableMetadata)42 PrestoException (com.facebook.presto.spi.PrestoException)42