Examples with Column com.huawei.boostkit.omnidata.model.Column used on opensource projects

Example 1 with Column

use of com.huawei.boostkit.omnidata.model.Column in project boostkit-bigdata by kunpengcompute.

the class PageSourceUtil method buildColumn.

private static Column buildColumn(HiveColumnHandle columnHandle, TypeManager typeManager, List<HivePartitionKey> partitionKeys, OptionalInt bucketNumber, Path path) {
    Type columnType = typeManager.getType(columnHandle.getTypeSignature());
    if (!columnHandle.getColumnType().equals(PARTITION_KEY)) {
        return new Column(columnHandle.getHiveColumnIndex(), columnHandle.getColumnName(), columnType);
    }
    Map<String, HivePartitionKey> partitionKeysMap = uniqueIndex(partitionKeys, HivePartitionKey::getName);
    String prefilledValue = HiveUtil.getPrefilledColumnValue(columnHandle, partitionKeysMap.get(columnHandle.getName()), path, bucketNumber);
    Object columnValue = HiveUtil.typedPartitionKey(prefilledValue, columnType, prefilledValue);
    return new Column(columnHandle.getHiveColumnIndex(), columnHandle.getColumnName(), columnType, true, columnValue);
}

Example 2 with Column

use of com.huawei.boostkit.omnidata.model.Column in project boostkit-bigdata by kunpengcompute.

the class DataIoAdapter method putFilterValue.

// column projection赋值
private int putFilterValue(Expression valueExpression, Type prestoType) {
    // Filter赋值
    int columnId = NdpUtils.getColumnId(valueExpression.toString()) - columnOffset;
    String filterColumnName = valueExpression.toString().split("#")[0];
    if (null != fieldMap.get(filterColumnName)) {
        return fieldMap.get(filterColumnName);
    }
    boolean isPartitionKey = partitionColumnName.contains(filterColumnName);
    int filterProjectionId = fieldMap.size();
    fieldMap.put(filterColumnName, filterProjectionId);
    filterTypesList.add(NdpUtils.transDataIoDataType(valueExpression.dataType()));
    filterOrdersList.add(filterProjectionId);
    String partitionValue = NdpUtils.getPartitionValue(filePath, filterColumnName);
    columnNameSet.add(filterColumnName);
    omnidataProjections.add(new InputReferenceExpression(filterProjectionId, prestoType));
    omnidataColumns.add(new Column(columnId, filterColumnName, prestoType, isPartitionKey, partitionValue));
    omnidataTypes.add(prestoType);
    if (null == columnNameMap.get(filterColumnName)) {
        columnNameMap.put(filterColumnName, columnNameMap.size());
    }
    return filterProjectionId;
}

Example 3 with Column

use of com.huawei.boostkit.omnidata.model.Column in project boostkit-bigdata by kunpengcompute.

the class DataIoAdapter method initColumnInfo.

private void initColumnInfo(Seq<Attribute> sparkOutPut) {
    if (listAtt == null || listAtt.size() == 0) {
        return;
    }
    List<Attribute> outputColumnList = JavaConverters.seqAsJavaList(sparkOutPut);
    boolean isPartitionKey;
    int filterColumnId = 0;
    for (Attribute attribute : outputColumnList) {
        Attribute resAttribute = NdpUtils.getColumnAttribute(attribute, listAtt);
        String columnName = resAttribute.name();
        Type type = NdpUtils.transOlkDataType(resAttribute.dataType(), false);
        int columnId = NdpUtils.getColumnId(resAttribute.toString()) - columnOffset;
        isPartitionKey = partitionColumnName.contains(columnName);
        String partitionValue = NdpUtils.getPartitionValue(filePath, columnName);
        omnidataColumns.add(new Column(columnId, columnName, type, isPartitionKey, partitionValue));
        omnidataTypes.add(type);
        filterTypesList.add(NdpUtils.transDataIoDataType(resAttribute.dataType()));
        filterOrdersList.add(filterColumnId);
        omnidataProjections.add(new InputReferenceExpression(filterColumnId, type));
        fieldMap.put(columnName, filterColumnId);
        ++filterColumnId;
    }
}

Example 4 with Column

use of com.huawei.boostkit.omnidata.model.Column in project boostkit-bigdata by kunpengcompute.

the class DataIoAdapter method createAggProjection.

private RowExpression createAggProjection(Expression expression) {
    Type prestoType = NdpUtils.transOlkDataType(expression.dataType(), false);
    AggExpressionType aggExpressionType = AggExpressionType.valueOf(expression.getClass().getSimpleName());
    switch(aggExpressionType) {
        case Add:
            return createAggBinCall((Add) expression, "Add", prestoType);
        case Subtract:
            return createAggBinCall((Subtract) expression, "Subtract", prestoType);
        case Multiply:
            return createAggBinCall((Multiply) expression, "Multiply", prestoType);
        case Divide:
            return createAggBinCall((Divide) expression, "Divide", prestoType);
        case Remainder:
            return createAggBinCall((Remainder) expression, "Remainder", prestoType);
        case Literal:
            Object value = NdpUtils.transData(expression.dataType().toString(), expression.toString());
            return new ConstantExpression(value, prestoType);
        case AttributeReference:
            String aggColumnName = expression.toString().split("#")[0];
            int field;
            if (null == columnNameMap.get(aggColumnName)) {
                field = columnNameMap.size();
                columnNameMap.put(aggColumnName, field);
                int columnId = NdpUtils.getColumnId(expression.toString()) - columnOffset;
                boolean isPartitionKey = partitionColumnName.contains(aggColumnName);
                String partitionValue = NdpUtils.getPartitionValue(filePath, aggColumnName);
                omnidataColumns.add(new Column(columnId, aggColumnName, prestoType, isPartitionKey, partitionValue));
            } else {
                field = columnNameMap.get(aggColumnName);
            }
            return new InputReferenceExpression(field, prestoType);
        default:
            throw new UnsupportedOperationException("unsupported agg operation type");
    }
}

Example 5 with Column

use of com.huawei.boostkit.omnidata.model.Column in project boostkit-bigdata by kunpengcompute.

the class DataIoAdapter method extractNamedExpression.

private RowExpression extractNamedExpression(Expression namedExpression) {
    Type prestoType = NdpUtils.transOlkDataType(namedExpression.dataType(), false);
    int aggProjectionId;
    String aggColumnName = namedExpression.toString().split("#")[0];
    columnOrdersList.add(columnOrder++);
    columnTypesList.add(NdpUtils.transDataIoDataType(namedExpression.dataType()));
    if (null != fieldMap.get(aggColumnName)) {
        aggProjectionId = fieldMap.get(aggColumnName);
    } else {
        int columnId = NdpUtils.getColumnId(namedExpression.toString()) - columnOffset;
        aggProjectionId = fieldMap.size();
        fieldMap.put(aggColumnName, aggProjectionId);
        omnidataTypes.add(prestoType);
        boolean isPartitionKey = partitionColumnName.contains(aggColumnName);
        String partitionValue = NdpUtils.getPartitionValue(filePath, aggColumnName);
        omnidataColumns.add(new Column(columnId, aggColumnName, prestoType, isPartitionKey, partitionValue));
        columnNameSet.add(aggColumnName);
        if (null == columnNameMap.get(aggColumnName)) {
            columnNameMap.put(aggColumnName, columnNameMap.size());
        }
        omnidataProjections.add(new InputReferenceExpression(aggProjectionId, prestoType));
    }
    return new InputReferenceExpression(aggProjectionId, prestoType);
}