Examples with KeyValueSchema org.apache.phoenix.schema.KeyValueSchema used on opensource projects

Example 1 with KeyValueSchema

use of org.apache.phoenix.schema.KeyValueSchema in project phoenix by apache.

the class ProjectionCompiler method compile.

/**
     * Builds the projection for the scan
     * @param context query context kept between compilation of different query clauses
     * @param statement TODO
     * @param groupBy compiled GROUP BY clause
     * @param targetColumns list of columns, parallel to aliasedNodes, that are being set for an
     * UPSERT SELECT statement. Used to coerce expression types to the expected target type.
     * @return projector used to access row values during scan
     * @throws SQLException 
     */
public static RowProjector compile(StatementContext context, SelectStatement statement, GroupBy groupBy, List<? extends PDatum> targetColumns, Expression where) throws SQLException {
    List<KeyValueColumnExpression> arrayKVRefs = new ArrayList<KeyValueColumnExpression>();
    List<ProjectedColumnExpression> arrayProjectedColumnRefs = new ArrayList<ProjectedColumnExpression>();
    List<Expression> arrayKVFuncs = new ArrayList<Expression>();
    List<Expression> arrayOldFuncs = new ArrayList<Expression>();
    Map<Expression, Integer> arrayExpressionCounts = new HashMap<>();
    List<AliasedNode> aliasedNodes = statement.getSelect();
    // Setup projected columns in Scan
    SelectClauseVisitor selectVisitor = new SelectClauseVisitor(context, groupBy, arrayKVRefs, arrayKVFuncs, arrayExpressionCounts, arrayProjectedColumnRefs, arrayOldFuncs, statement);
    List<ExpressionProjector> projectedColumns = new ArrayList<ExpressionProjector>();
    ColumnResolver resolver = context.getResolver();
    TableRef tableRef = context.getCurrentTable();
    PTable table = tableRef.getTable();
    boolean resolveColumn = !tableRef.equals(resolver.getTables().get(0));
    boolean isWildcard = false;
    Scan scan = context.getScan();
    int index = 0;
    List<Expression> projectedExpressions = Lists.newArrayListWithExpectedSize(aliasedNodes.size());
    List<byte[]> projectedFamilies = Lists.newArrayListWithExpectedSize(aliasedNodes.size());
    for (AliasedNode aliasedNode : aliasedNodes) {
        ParseNode node = aliasedNode.getNode();
        // TODO: visitor?
        if (node instanceof WildcardParseNode) {
            if (statement.isAggregate()) {
                ExpressionCompiler.throwNonAggExpressionInAggException(node.toString());
            }
            if (tableRef == TableRef.EMPTY_TABLE_REF) {
                throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_TABLE_SPECIFIED_FOR_WILDCARD_SELECT).build().buildException();
            }
            isWildcard = true;
            if (tableRef.getTable().getType() == PTableType.INDEX && ((WildcardParseNode) node).isRewrite()) {
                projectAllIndexColumns(context, tableRef, resolveColumn, projectedExpressions, projectedColumns, targetColumns);
            } else {
                projectAllTableColumns(context, tableRef, resolveColumn, projectedExpressions, projectedColumns, targetColumns);
            }
        } else if (node instanceof TableWildcardParseNode) {
            TableName tName = ((TableWildcardParseNode) node).getTableName();
            TableRef tRef = resolver.resolveTable(tName.getSchemaName(), tName.getTableName());
            if (tRef.equals(tableRef)) {
                isWildcard = true;
            }
            if (tRef.getTable().getType() == PTableType.INDEX && ((TableWildcardParseNode) node).isRewrite()) {
                projectAllIndexColumns(context, tRef, true, projectedExpressions, projectedColumns, targetColumns);
            } else {
                projectAllTableColumns(context, tRef, true, projectedExpressions, projectedColumns, targetColumns);
            }
        } else if (node instanceof FamilyWildcardParseNode) {
            if (tableRef == TableRef.EMPTY_TABLE_REF) {
                throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_TABLE_SPECIFIED_FOR_WILDCARD_SELECT).build().buildException();
            }
            // Project everything for SELECT cf.*
            String cfName = ((FamilyWildcardParseNode) node).getName();
            // Delay projecting to scan, as when any other column in the column family gets
            // added to the scan, it overwrites that we want to project the entire column
            // family. Instead, we do the projection at the end.
            // TODO: consider having a ScanUtil.addColumn and ScanUtil.addFamily to work
            // around this, as this code depends on this function being the last place where
            // columns are projected (which is currently true, but could change).
            projectedFamilies.add(Bytes.toBytes(cfName));
            if (tableRef.getTable().getType() == PTableType.INDEX && ((FamilyWildcardParseNode) node).isRewrite()) {
                projectIndexColumnFamily(context, cfName, tableRef, resolveColumn, projectedExpressions, projectedColumns);
            } else {
                projectTableColumnFamily(context, cfName, tableRef, resolveColumn, projectedExpressions, projectedColumns);
            }
        } else {
            Expression expression = node.accept(selectVisitor);
            projectedExpressions.add(expression);
            expression = coerceIfNecessary(index, targetColumns, expression);
            if (node instanceof BindParseNode) {
                context.getBindManager().addParamMetaData((BindParseNode) node, expression);
            }
            if (!node.isStateless()) {
                if (!selectVisitor.isAggregate() && statement.isAggregate()) {
                    ExpressionCompiler.throwNonAggExpressionInAggException(expression.toString());
                }
            }
            String columnAlias = aliasedNode.getAlias() != null ? aliasedNode.getAlias() : SchemaUtil.normalizeIdentifier(aliasedNode.getNode().getAlias());
            boolean isCaseSensitive = aliasedNode.getAlias() != null ? aliasedNode.isCaseSensitve() : (columnAlias != null ? SchemaUtil.isCaseSensitive(aliasedNode.getNode().getAlias()) : selectVisitor.isCaseSensitive);
            String name = columnAlias == null ? expression.toString() : columnAlias;
            projectedColumns.add(new ExpressionProjector(name, tableRef.getTableAlias() == null ? (table.getName() == null ? "" : table.getName().getString()) : tableRef.getTableAlias(), expression, isCaseSensitive));
        }
        selectVisitor.reset();
        index++;
    }
    for (int i = arrayProjectedColumnRefs.size() - 1; i >= 0; i--) {
        Expression expression = arrayProjectedColumnRefs.get(i);
        Integer count = arrayExpressionCounts.get(expression);
        if (count != 0) {
            arrayKVRefs.remove(i);
            arrayKVFuncs.remove(i);
            arrayOldFuncs.remove(i);
        }
    }
    if (arrayKVFuncs.size() > 0 && arrayKVRefs.size() > 0) {
        serailizeArrayIndexInformationAndSetInScan(context, arrayKVFuncs, arrayKVRefs);
        KeyValueSchemaBuilder builder = new KeyValueSchemaBuilder(0);
        for (Expression expression : arrayKVRefs) {
            builder.addField(expression);
        }
        KeyValueSchema kvSchema = builder.build();
        ValueBitSet arrayIndexesBitSet = ValueBitSet.newInstance(kvSchema);
        builder = new KeyValueSchemaBuilder(0);
        for (Expression expression : arrayKVFuncs) {
            builder.addField(expression);
        }
        KeyValueSchema arrayIndexesSchema = builder.build();
        Map<Expression, Expression> replacementMap = new HashMap<>();
        for (int i = 0; i < arrayOldFuncs.size(); i++) {
            Expression function = arrayKVFuncs.get(i);
            replacementMap.put(arrayOldFuncs.get(i), new ArrayIndexExpression(i, function.getDataType(), arrayIndexesBitSet, arrayIndexesSchema));
        }
        ReplaceArrayFunctionExpressionVisitor visitor = new ReplaceArrayFunctionExpressionVisitor(replacementMap);
        for (int i = 0; i < projectedColumns.size(); i++) {
            ExpressionProjector projector = projectedColumns.get(i);
            projectedColumns.set(i, new ExpressionProjector(projector.getName(), tableRef.getTableAlias() == null ? (table.getName() == null ? "" : table.getName().getString()) : tableRef.getTableAlias(), projector.getExpression().accept(visitor), projector.isCaseSensitive()));
        }
    }
    // TODO make estimatedByteSize more accurate by counting the joined columns.
    int estimatedKeySize = table.getRowKeySchema().getEstimatedValueLength();
    int estimatedByteSize = 0;
    for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
        PColumnFamily family = table.getColumnFamily(entry.getKey());
        if (entry.getValue() == null) {
            for (PColumn column : family.getColumns()) {
                Integer maxLength = column.getMaxLength();
                int byteSize = column.getDataType().isFixedWidth() ? maxLength == null ? column.getDataType().getByteSize() : maxLength : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE;
                estimatedByteSize += SizedUtil.KEY_VALUE_SIZE + estimatedKeySize + byteSize;
            }
        } else {
            for (byte[] cq : entry.getValue()) {
                //if (!Bytes.equals(cq, ByteUtil.EMPTY_BYTE_ARRAY) || cq.length > 0) {
                PColumn column = family.getPColumnForColumnQualifier(cq);
                Integer maxLength = column.getMaxLength();
                int byteSize = column.getDataType().isFixedWidth() ? maxLength == null ? column.getDataType().getByteSize() : maxLength : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE;
                estimatedByteSize += SizedUtil.KEY_VALUE_SIZE + estimatedKeySize + byteSize;
            }
        //}
        }
    }
    boolean isProjectEmptyKeyValue = false;
    if (isWildcard) {
        projectAllColumnFamilies(table, scan);
    } else {
        isProjectEmptyKeyValue = where == null || LiteralExpression.isTrue(where) || where.requiresFinalEvaluation();
        for (byte[] family : projectedFamilies) {
            projectColumnFamily(table, scan, family);
        }
    }
    return new RowProjector(projectedColumns, estimatedByteSize, isProjectEmptyKeyValue, resolver.hasUDFs(), isWildcard);
}

Example 2 with KeyValueSchema

use of org.apache.phoenix.schema.KeyValueSchema in project phoenix by apache.

the class ProjectedColumnExpression method evaluate.

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    try {
        KeyValueSchema schema = getSchema();
        TupleProjector.decodeProjectedValue(tuple, ptr);
        bitSet.clear();
        bitSet.or(ptr);
        int maxOffset = ptr.getOffset() + ptr.getLength() - bitSet.getEstimatedLength();
        schema.iterator(ptr, position, bitSet);
        Boolean hasValue = schema.next(ptr, position, maxOffset, bitSet);
        if (hasValue == null || !hasValue.booleanValue())
            return false;
    } catch (IOException e) {
        return false;
    }
    return true;
}

Example 3 with KeyValueSchema

use of org.apache.phoenix.schema.KeyValueSchema in project phoenix by apache.

the class IndexUtil method deserializeLocalIndexJoinSchemaFromScan.

public static KeyValueSchema deserializeLocalIndexJoinSchemaFromScan(final Scan scan) {
    byte[] schemaBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_JOIN_SCHEMA);
    if (schemaBytes == null)
        return null;
    // TODO: size?
    ByteArrayInputStream stream = new ByteArrayInputStream(schemaBytes);
    try {
        DataInputStream input = new DataInputStream(stream);
        KeyValueSchema schema = new KeyValueSchema();
        schema.readFields(input);
        return schema;
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        try {
            stream.close();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
}

Example 4 with KeyValueSchema

use of org.apache.phoenix.schema.KeyValueSchema in project phoenix by apache.

the class IndexUtil method getTupleProjector.

public static TupleProjector getTupleProjector(Scan scan, ColumnReference[] dataColumns) {
    if (dataColumns != null && dataColumns.length != 0) {
        KeyValueSchema keyValueSchema = deserializeLocalIndexJoinSchemaFromScan(scan);
        boolean storeColsInSingleCell = scan.getAttribute(BaseScannerRegionObserver.COLUMNS_STORED_IN_SINGLE_CELL) != null;
        QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);
        Expression[] colExpressions = storeColsInSingleCell ? new SingleCellColumnExpression[dataColumns.length] : new KeyValueColumnExpression[dataColumns.length];
        for (int i = 0; i < dataColumns.length; i++) {
            byte[] family = dataColumns[i].getFamily();
            byte[] qualifier = dataColumns[i].getQualifier();
            Field field = keyValueSchema.getField(i);
            Expression dataColumnExpr = storeColsInSingleCell ? new SingleCellColumnExpression(field, family, qualifier, encodingScheme) : new KeyValueColumnExpression(field, family, qualifier);
            colExpressions[i] = dataColumnExpr;
        }
        return new TupleProjector(keyValueSchema, colExpressions);
    }
    return null;
}

Example 5 with KeyValueSchema

use of org.apache.phoenix.schema.KeyValueSchema in project phoenix by apache.

the class PhoenixRuntime method encodeColumnValues.

/**
     * 
     * @param conn connection that was used for reading/generating value.
     * @param fullTableName fully qualified table name
     * @param values values of the columns
     * @param columns list of pair of column that includes column family as first part and column name as the second part.
     * Column family is optional and hence nullable. Columns in the list have to be in the same order as the order of occurence
     * of their values in the object array.
     * @return values encoded in a byte array 
     * @throws SQLException
     * @see {@link #decodeValues(Connection, String, byte[], List)}
     */
public static byte[] encodeColumnValues(Connection conn, String fullTableName, Object[] values, List<Pair<String, String>> columns) throws SQLException {
    PTable table = getTable(conn, fullTableName);
    List<PColumn> pColumns = getColumns(table, columns);
    List<Expression> expressions = new ArrayList<Expression>(pColumns.size());
    int i = 0;
    for (PColumn col : pColumns) {
        Object value = values[i];
        // for purposes of encoding, sort order of the columns doesn't matter.
        Expression expr = LiteralExpression.newConstant(value, col.getDataType(), col.getMaxLength(), col.getScale());
        expressions.add(expr);
        i++;
    }
    KeyValueSchema kvSchema = buildKeyValueSchema(pColumns);
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    ValueBitSet valueSet = ValueBitSet.newInstance(kvSchema);
    return kvSchema.toBytes(expressions.toArray(new Expression[0]), valueSet, ptr);
}