Examples with RowType com.facebook.presto.common.type.RowType used on opensource projects

Example 46 with RowType

use of com.facebook.presto.common.type.RowType in project presto by prestodb.

the class MultimapFromEntriesFunction method multimapFromEntries.

@TypeParameter("K")
@TypeParameter("V")
@SqlType("map(K,array(V))")
@SqlNullable
public Block multimapFromEntries(@TypeParameter("map(K,array(V))") MapType mapType, @SqlType("array(row(K,V))") Block block) {
    Type keyType = mapType.getKeyType();
    Type valueType = ((ArrayType) mapType.getValueType()).getElementType();
    RowType rowType = RowType.anonymous(ImmutableList.of(keyType, valueType));
    int entryCount = block.getPositionCount();
    IntList[] entryIndicesList = new IntList[entryCount];
    for (int i = 0; i < entryIndicesList.length; i++) {
        entryIndicesList[i] = new IntArrayList();
    }
    TypedSet keySet = new TypedSet(keyType, entryCount, NAME);
    for (int i = 0; i < entryCount; i++) {
        if (block.isNull(i)) {
            throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "map entry cannot be null");
        }
        Block rowBlock = rowType.getObject(block, i);
        if (rowBlock.isNull(0)) {
            throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "map key cannot be null");
        }
        if (keySet.contains(rowBlock, 0)) {
            entryIndicesList[keySet.positionOf(rowBlock, 0)].add(i);
        } else {
            keySet.add(rowBlock, 0);
            entryIndicesList[keySet.size() - 1].add(i);
        }
    }
    BlockBuilder multimapBlockBuilder = mapType.createBlockBuilder(null, keySet.size());
    BlockBuilder singleMapWriter = multimapBlockBuilder.beginBlockEntry();
    for (int i = 0; i < keySet.size(); i++) {
        keyType.appendTo(rowType.getObject(block, entryIndicesList[i].getInt(0)), 0, singleMapWriter);
        BlockBuilder singleArrayWriter = singleMapWriter.beginBlockEntry();
        for (int entryIndex : entryIndicesList[i]) {
            valueType.appendTo(rowType.getObject(block, entryIndex), 1, singleArrayWriter);
        }
        singleMapWriter.closeEntry();
    }
    multimapBlockBuilder.closeEntry();
    return mapType.getObject(multimapBlockBuilder, multimapBlockBuilder.getPositionCount() - 1);
}

Example 47 with RowType

use of com.facebook.presto.common.type.RowType in project presto by prestodb.

the class ParquetDereferencePushDown method createNestedColumn.

private Subfield createNestedColumn(Map<String, ColumnHandle> baseColumnHandles, RowExpression rowExpression, ExpressionOptimizer expressionOptimizer, ConnectorSession session) {
    if (!(rowExpression instanceof SpecialFormExpression) || ((SpecialFormExpression) rowExpression).getForm() != DEREFERENCE) {
        throw new IllegalArgumentException("expecting SpecialFormExpression(DEREFERENCE), but got: " + rowExpression);
    }
    RowExpression currentRowExpression = rowExpression;
    List<Subfield.PathElement> elements = new ArrayList<>();
    while (true) {
        if (currentRowExpression instanceof VariableReferenceExpression) {
            Collections.reverse(elements);
            String name = ((VariableReferenceExpression) currentRowExpression).getName();
            ColumnHandle handle = baseColumnHandles.get(name);
            checkArgument(handle != null, "Missing Column handle: " + name);
            String originalColumnName = getColumnName(handle);
            return new Subfield(originalColumnName, unmodifiableList(elements));
        }
        if (currentRowExpression instanceof SpecialFormExpression && ((SpecialFormExpression) currentRowExpression).getForm() == DEREFERENCE) {
            SpecialFormExpression dereferenceExpression = (SpecialFormExpression) currentRowExpression;
            RowExpression base = dereferenceExpression.getArguments().get(0);
            RowType baseType = (RowType) base.getType();
            RowExpression indexExpression = expressionOptimizer.optimize(dereferenceExpression.getArguments().get(1), ExpressionOptimizer.Level.OPTIMIZED, session);
            if (indexExpression instanceof ConstantExpression) {
                Object index = ((ConstantExpression) indexExpression).getValue();
                if (index instanceof Number) {
                    Optional<String> fieldName = baseType.getFields().get(((Number) index).intValue()).getName();
                    if (fieldName.isPresent()) {
                        elements.add(new Subfield.NestedField(fieldName.get()));
                        currentRowExpression = base;
                        continue;
                    }
                }
            }
        }
        break;
    }
    throw new IllegalArgumentException("expecting SpecialFormExpression(DEREFERENCE) with constants for indices, but got: " + currentRowExpression);
}

Example 48 with RowType

use of com.facebook.presto.common.type.RowType in project presto by prestodb.

the class MaterializedResult method writeValue.

private static void writeValue(Type type, BlockBuilder blockBuilder, Object value) {
    if (value == null) {
        blockBuilder.appendNull();
    } else if (BIGINT.equals(type)) {
        type.writeLong(blockBuilder, ((Number) value).longValue());
    } else if (INTEGER.equals(type)) {
        type.writeLong(blockBuilder, ((Number) value).intValue());
    } else if (SMALLINT.equals(type)) {
        type.writeLong(blockBuilder, ((Number) value).shortValue());
    } else if (TINYINT.equals(type)) {
        type.writeLong(blockBuilder, ((Number) value).byteValue());
    } else if (REAL.equals(type)) {
        type.writeLong(blockBuilder, (long) floatToRawIntBits(((Number) value).floatValue()));
    } else if (DOUBLE.equals(type)) {
        type.writeDouble(blockBuilder, ((Number) value).doubleValue());
    } else if (BOOLEAN.equals(type)) {
        type.writeBoolean(blockBuilder, (Boolean) value);
    } else if (type instanceof VarcharType) {
        type.writeSlice(blockBuilder, Slices.utf8Slice((String) value));
    } else if (type instanceof CharType) {
        type.writeSlice(blockBuilder, Slices.utf8Slice((String) value));
    } else if (VARBINARY.equals(type)) {
        type.writeSlice(blockBuilder, Slices.wrappedBuffer((byte[]) value));
    } else if (DATE.equals(type)) {
        int days = ((SqlDate) value).getDays();
        type.writeLong(blockBuilder, days);
    } else if (TIME.equals(type)) {
        SqlTime time = (SqlTime) value;
        if (time.isLegacyTimestamp()) {
            type.writeLong(blockBuilder, time.getMillisUtc());
        } else {
            type.writeLong(blockBuilder, time.getMillis());
        }
    } else if (TIME_WITH_TIME_ZONE.equals(type)) {
        long millisUtc = ((SqlTimeWithTimeZone) value).getMillisUtc();
        TimeZoneKey timeZoneKey = ((SqlTimeWithTimeZone) value).getTimeZoneKey();
        type.writeLong(blockBuilder, packDateTimeWithZone(millisUtc, timeZoneKey));
    } else if (TIMESTAMP.equals(type)) {
        long millisUtc = ((SqlTimestamp) value).getMillisUtc();
        type.writeLong(blockBuilder, millisUtc);
    } else if (TIMESTAMP_WITH_TIME_ZONE.equals(type)) {
        long millisUtc = ((SqlTimestampWithTimeZone) value).getMillisUtc();
        TimeZoneKey timeZoneKey = ((SqlTimestampWithTimeZone) value).getTimeZoneKey();
        type.writeLong(blockBuilder, packDateTimeWithZone(millisUtc, timeZoneKey));
    } else if (ARRAY.equals(type.getTypeSignature().getBase())) {
        List<Object> list = (List<Object>) value;
        Type elementType = ((ArrayType) type).getElementType();
        BlockBuilder arrayBlockBuilder = blockBuilder.beginBlockEntry();
        for (Object element : list) {
            writeValue(elementType, arrayBlockBuilder, element);
        }
        blockBuilder.closeEntry();
    } else if (MAP.equals(type.getTypeSignature().getBase())) {
        Map<Object, Object> map = (Map<Object, Object>) value;
        Type keyType = ((MapType) type).getKeyType();
        Type valueType = ((MapType) type).getValueType();
        BlockBuilder mapBlockBuilder = blockBuilder.beginBlockEntry();
        for (Entry<Object, Object> entry : map.entrySet()) {
            writeValue(keyType, mapBlockBuilder, entry.getKey());
            writeValue(valueType, mapBlockBuilder, entry.getValue());
        }
        blockBuilder.closeEntry();
    } else if (type instanceof RowType) {
        List<Object> row = (List<Object>) value;
        List<Type> fieldTypes = type.getTypeParameters();
        BlockBuilder rowBlockBuilder = blockBuilder.beginBlockEntry();
        for (int field = 0; field < row.size(); field++) {
            writeValue(fieldTypes.get(field), rowBlockBuilder, row.get(field));
        }
        blockBuilder.closeEntry();
    } else {
        throw new IllegalArgumentException("Unsupported type " + type);
    }
}

Example 49 with RowType

use of com.facebook.presto.common.type.RowType in project presto by prestodb.

the class HiveMetadata method getTableEncryptionPropertiesFromTableProperties.

protected Optional<TableEncryptionProperties> getTableEncryptionPropertiesFromTableProperties(ConnectorTableMetadata tableMetadata, HiveStorageFormat hiveStorageFormat, List<String> partitionedBy) {
    ColumnEncryptionInformation columnEncryptionInformation = getEncryptColumns(tableMetadata.getProperties());
    String tableEncryptionReference = getEncryptTable(tableMetadata.getProperties());
    if (tableEncryptionReference == null && (columnEncryptionInformation == null || !columnEncryptionInformation.hasEntries())) {
        return Optional.empty();
    }
    if (tableEncryptionReference != null && columnEncryptionInformation.hasEntries()) {
        throw new PrestoException(INVALID_TABLE_PROPERTY, format("Only one of %s or %s should be specified", ENCRYPT_TABLE, ENCRYPT_COLUMNS));
    }
    if (hiveStorageFormat != DWRF) {
        throw new PrestoException(NOT_SUPPORTED, "Only DWRF file format supports encryption at this time");
    }
    if (tableEncryptionReference != null) {
        return Optional.of(getDwrfTableEncryptionProperties(Optional.of(tableEncryptionReference), Optional.empty(), tableMetadata));
    }
    partitionedBy.forEach(partitionColumn -> {
        if (columnEncryptionInformation.getColumnToKeyReference().containsKey(ColumnWithStructSubfield.valueOf(partitionColumn))) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, format("Partition column (%s) cannot be used as an encryption column", partitionColumn));
        }
    });
    Map<String, ColumnMetadata> columnMetadata = tableMetadata.getColumns().stream().collect(toImmutableMap(ColumnMetadata::getName, identity()));
    // Sorting to ensure that we can find cases of multiple referenceKeys within the same struct chain. Example - a.b and a.b.c
    // By sorting we ensure that we have already seen a.b and can find that when we visit a.b.c
    List<ColumnWithStructSubfield> sortedColumns = new ArrayList<>(columnEncryptionInformation.getColumnToKeyReference().keySet());
    sortedColumns.sort(Comparator.comparing(ColumnWithStructSubfield::toString));
    Set<String> seenColumns = new HashSet<>();
    for (ColumnWithStructSubfield columnWithSubfield : sortedColumns) {
        ColumnMetadata column = columnMetadata.get(columnWithSubfield.getColumnName());
        if (column == null) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, format("In %s unable to find column %s", ENCRYPT_COLUMNS, columnWithSubfield.getColumnName()));
        }
        if (seenColumns.contains(columnWithSubfield.toString())) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, format("The same column/subfield cannot have 2 encryption keys"));
        }
        if (columnWithSubfield.getSubfieldPath().isPresent()) {
            Iterable<String> subfieldPathFragments = Splitter.on(".").split(columnWithSubfield.getSubfieldPath().get());
            Type columnType = column.getType();
            String parentPath = columnWithSubfield.getColumnName();
            for (String pathFragment : subfieldPathFragments) {
                if (!(columnType instanceof RowType)) {
                    throw new PrestoException(INVALID_TABLE_PROPERTY, format("In %s subfields declared in %s, but %s has type %s", ENCRYPT_COLUMNS, columnWithSubfield.toString(), column.getName(), column.getType().getDisplayName()));
                }
                if (seenColumns.contains(parentPath)) {
                    throw new PrestoException(INVALID_TABLE_PROPERTY, format("For (%s) found a keyReference at a higher level field (%s)", columnWithSubfield.toString(), parentPath));
                }
                RowType row = (RowType) columnType;
                columnType = row.getFields().stream().filter(f -> f.getName().orElse("").equals(pathFragment)).findAny().map(RowType.Field::getType).orElseThrow(() -> new PrestoException(INVALID_TABLE_PROPERTY, format("In %s subfields declared in %s, but %s has type %s", ENCRYPT_COLUMNS, columnWithSubfield.toString(), column.getName(), column.getType().getDisplayName())));
                parentPath = format("%s.%s", parentPath, pathFragment);
            }
        }
        seenColumns.add(columnWithSubfield.toString());
    }
    return Optional.of(getDwrfTableEncryptionProperties(Optional.empty(), Optional.of(columnEncryptionInformation), tableMetadata));
}

Example 50 with RowType

use of com.facebook.presto.common.type.RowType in project presto by prestodb.

the class SubfieldExtractor method toSubfield.

private static Optional<Subfield> toSubfield(RowExpression expression, StandardFunctionResolution functionResolution, ExpressionOptimizer expressionOptimizer, ConnectorSession connectorSession) {
    List<Subfield.PathElement> elements = new ArrayList<>();
    while (true) {
        if (expression instanceof VariableReferenceExpression) {
            Collections.reverse(elements);
            return Optional.of(new Subfield(((VariableReferenceExpression) expression).getName(), unmodifiableList(elements)));
        }
        if (expression instanceof SpecialFormExpression && ((SpecialFormExpression) expression).getForm() == DEREFERENCE) {
            SpecialFormExpression dereferenceExpression = (SpecialFormExpression) expression;
            RowExpression base = dereferenceExpression.getArguments().get(0);
            RowType baseType = (RowType) base.getType();
            RowExpression indexExpression = expressionOptimizer.optimize(dereferenceExpression.getArguments().get(1), ExpressionOptimizer.Level.OPTIMIZED, connectorSession);
            if (indexExpression instanceof ConstantExpression) {
                Object index = ((ConstantExpression) indexExpression).getValue();
                if (index instanceof Number) {
                    Optional<String> fieldName = baseType.getFields().get(((Number) index).intValue()).getName();
                    if (fieldName.isPresent()) {
                        elements.add(new Subfield.NestedField(fieldName.get()));
                        expression = base;
                        continue;
                    }
                }
            }
            return Optional.empty();
        }
        if (expression instanceof CallExpression && functionResolution.isSubscriptFunction(((CallExpression) expression).getFunctionHandle())) {
            List<RowExpression> arguments = ((CallExpression) expression).getArguments();
            RowExpression indexExpression = expressionOptimizer.optimize(arguments.get(1), ExpressionOptimizer.Level.OPTIMIZED, connectorSession);
            if (indexExpression instanceof ConstantExpression) {
                Object index = ((ConstantExpression) indexExpression).getValue();
                if (index instanceof Number) {
                    elements.add(new Subfield.LongSubscript(((Number) index).longValue()));
                    expression = arguments.get(0);
                    continue;
                }
                if (isVarcharType(indexExpression.getType())) {
                    elements.add(new Subfield.StringSubscript(((Slice) index).toStringUtf8()));
                    expression = arguments.get(0);
                    continue;
                }
            }
            return Optional.empty();
        }
        return Optional.empty();
    }
}