Examples with KeyValueColumnExpression org.apache.phoenix.expression.KeyValueColumnExpression used on opensource projects

Example 11 with KeyValueColumnExpression

use of org.apache.phoenix.expression.KeyValueColumnExpression in project phoenix by apache.

the class RegionScannerFactory method getWrappedScanner.

/**
   * Return wrapped scanner that catches unexpected exceptions (i.e. Phoenix bugs) and
   * re-throws as DoNotRetryIOException to prevent needless retrying hanging the query
   * for 30 seconds. Unfortunately, until HBASE-7481 gets fixed, there's no way to do
   * the same from a custom filter.
   * @param arrayKVRefs
   * @param arrayFuncRefs
   * @param offset starting position in the rowkey.
   * @param scan
   * @param tupleProjector
   * @param dataRegion
   * @param indexMaintainer
   * @param tx current transaction
   * @param viewConstants
   */
public RegionScanner getWrappedScanner(final RegionCoprocessorEnvironment env, final RegionScanner s, final Set<KeyValueColumnExpression> arrayKVRefs, final Expression[] arrayFuncRefs, final int offset, final Scan scan, final ColumnReference[] dataColumns, final TupleProjector tupleProjector, final Region dataRegion, final IndexMaintainer indexMaintainer, Transaction tx, final byte[][] viewConstants, final KeyValueSchema kvSchema, final ValueBitSet kvSchemaBitSet, final TupleProjector projector, final ImmutableBytesWritable ptr, final boolean useQualifierAsListIndex) {
    return new RegionScanner() {

        private boolean hasReferences = checkForReferenceFiles();

        private HRegionInfo regionInfo = env.getRegionInfo();

        private byte[] actualStartKey = getActualStartKey();

        // If there are any reference files after local index region merge some cases we might
        // get the records less than scan start row key. This will happen when we replace the
        // actual region start key with merge region start key. This method gives whether are
        // there any reference files in the region or not.
        private boolean checkForReferenceFiles() {
            if (!ScanUtil.isLocalIndex(scan))
                return false;
            for (byte[] family : scan.getFamilies()) {
                if (getRegion().getStore(family).hasReferences()) {
                    return true;
                }
            }
            return false;
        }

        // Get the actual scan start row of local index. This will be used to compare the row
        // key of the results less than scan start row when there are references.
        public byte[] getActualStartKey() {
            return ScanUtil.isLocalIndex(scan) ? ScanUtil.getActualStartRow(scan, regionInfo) : null;
        }

        @Override
        public boolean next(List<Cell> results) throws IOException {
            try {
                return s.next(results);
            } catch (Throwable t) {
                ServerUtil.throwIOException(getRegion().getRegionInfo().getRegionNameAsString(), t);
                // impossible
                return false;
            }
        }

        @Override
        public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
            try {
                return s.next(result, scannerContext);
            } catch (Throwable t) {
                ServerUtil.throwIOException(getRegion().getRegionInfo().getRegionNameAsString(), t);
                // impossible
                return false;
            }
        }

        @Override
        public void close() throws IOException {
            s.close();
        }

        @Override
        public HRegionInfo getRegionInfo() {
            return s.getRegionInfo();
        }

        @Override
        public boolean isFilterDone() throws IOException {
            return s.isFilterDone();
        }

        @Override
        public boolean reseek(byte[] row) throws IOException {
            return s.reseek(row);
        }

        @Override
        public long getMvccReadPoint() {
            return s.getMvccReadPoint();
        }

        @Override
        public boolean nextRaw(List<Cell> result) throws IOException {
            try {
                boolean next = s.nextRaw(result);
                Cell arrayElementCell = null;
                if (result.size() == 0) {
                    return next;
                }
                if (arrayFuncRefs != null && arrayFuncRefs.length > 0 && arrayKVRefs.size() > 0) {
                    int arrayElementCellPosition = replaceArrayIndexElement(arrayKVRefs, arrayFuncRefs, result);
                    arrayElementCell = result.get(arrayElementCellPosition);
                }
                if (ScanUtil.isLocalIndex(scan) && !ScanUtil.isAnalyzeTable(scan)) {
                    if (hasReferences && actualStartKey != null) {
                        next = scanTillScanStartRow(s, arrayKVRefs, arrayFuncRefs, result, null, arrayElementCell);
                        if (result.isEmpty()) {
                            return next;
                        }
                    }
                    /* In the following, c is only used when data region is null.
            dataRegion will never be null in case of non-coprocessor call,
            therefore no need to refactor
             */
                    IndexUtil.wrapResultUsingOffset(env, result, offset, dataColumns, tupleProjector, dataRegion, indexMaintainer, viewConstants, ptr);
                }
                if (projector != null) {
                    Tuple toProject = useQualifierAsListIndex ? new PositionBasedResultTuple(result) : new ResultTuple(Result.create(result));
                    Tuple tuple = projector.projectResults(toProject, useNewValueColumnQualifier);
                    result.clear();
                    result.add(tuple.getValue(0));
                    if (arrayElementCell != null) {
                        result.add(arrayElementCell);
                    }
                }
                // There is a scanattribute set to retrieve the specific array element
                return next;
            } catch (Throwable t) {
                ServerUtil.throwIOException(getRegion().getRegionInfo().getRegionNameAsString(), t);
                // impossible
                return false;
            }
        }

        @Override
        public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
            try {
                boolean next = s.nextRaw(result, scannerContext);
                Cell arrayElementCell = null;
                if (result.size() == 0) {
                    return next;
                }
                if (arrayFuncRefs != null && arrayFuncRefs.length > 0 && arrayKVRefs.size() > 0) {
                    int arrayElementCellPosition = replaceArrayIndexElement(arrayKVRefs, arrayFuncRefs, result);
                    arrayElementCell = result.get(arrayElementCellPosition);
                }
                if ((offset > 0 || ScanUtil.isLocalIndex(scan)) && !ScanUtil.isAnalyzeTable(scan)) {
                    if (hasReferences && actualStartKey != null) {
                        next = scanTillScanStartRow(s, arrayKVRefs, arrayFuncRefs, result, scannerContext, arrayElementCell);
                        if (result.isEmpty()) {
                            return next;
                        }
                    }
                    /* In the following, c is only used when data region is null.
            dataRegion will never be null in case of non-coprocessor call,
            therefore no need to refactor
             */
                    IndexUtil.wrapResultUsingOffset(env, result, offset, dataColumns, tupleProjector, dataRegion, indexMaintainer, viewConstants, ptr);
                }
                if (projector != null) {
                    Tuple toProject = useQualifierAsListIndex ? new PositionBasedMultiKeyValueTuple(result) : new ResultTuple(Result.create(result));
                    Tuple tuple = projector.projectResults(toProject, useNewValueColumnQualifier);
                    result.clear();
                    result.add(tuple.getValue(0));
                    if (arrayElementCell != null)
                        result.add(arrayElementCell);
                }
                // There is a scanattribute set to retrieve the specific array element
                return next;
            } catch (Throwable t) {
                ServerUtil.throwIOException(getRegion().getRegionInfo().getRegionNameAsString(), t);
                // impossible
                return false;
            }
        }

        /**
       * When there is a merge in progress while scanning local indexes we might get the key values less than scan start row.
       * In that case we need to scan until get the row key more or  equal to scan start key.
       * TODO try to fix this case in LocalIndexStoreFileScanner when there is a merge.
       */
        private boolean scanTillScanStartRow(final RegionScanner s, final Set<KeyValueColumnExpression> arrayKVRefs, final Expression[] arrayFuncRefs, List<Cell> result, ScannerContext scannerContext, Cell arrayElementCell) throws IOException {
            boolean next = true;
            Cell firstCell = result.get(0);
            while (Bytes.compareTo(firstCell.getRowArray(), firstCell.getRowOffset(), firstCell.getRowLength(), actualStartKey, 0, actualStartKey.length) < 0) {
                result.clear();
                if (scannerContext == null) {
                    next = s.nextRaw(result);
                } else {
                    next = s.nextRaw(result, scannerContext);
                }
                if (result.isEmpty()) {
                    return next;
                }
                if (arrayFuncRefs != null && arrayFuncRefs.length > 0 && arrayKVRefs.size() > 0) {
                    int arrayElementCellPosition = replaceArrayIndexElement(arrayKVRefs, arrayFuncRefs, result);
                    arrayElementCell = result.get(arrayElementCellPosition);
                }
                firstCell = result.get(0);
            }
            return next;
        }

        private int replaceArrayIndexElement(final Set<KeyValueColumnExpression> arrayKVRefs, final Expression[] arrayFuncRefs, List<Cell> result) {
            // make a copy of the results array here, as we're modifying it below
            MultiKeyValueTuple tuple = new MultiKeyValueTuple(ImmutableList.copyOf(result));
            // The size of both the arrays would be same?
            // Using KeyValueSchema to set and retrieve the value
            // collect the first kv to get the row
            Cell rowKv = result.get(0);
            for (KeyValueColumnExpression kvExp : arrayKVRefs) {
                if (kvExp.evaluate(tuple, ptr)) {
                    ListIterator<Cell> itr = result.listIterator();
                    while (itr.hasNext()) {
                        Cell kv = itr.next();
                        if (Bytes.equals(kvExp.getColumnFamily(), 0, kvExp.getColumnFamily().length, kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength()) && Bytes.equals(kvExp.getColumnQualifier(), 0, kvExp.getColumnQualifier().length, kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength())) {
                            // remove the kv that has the full array values.
                            itr.remove();
                            break;
                        }
                    }
                }
            }
            byte[] value = kvSchema.toBytes(tuple, arrayFuncRefs, kvSchemaBitSet, ptr);
            // Add a dummy kv with the exact value of the array index
            result.add(new KeyValue(rowKv.getRowArray(), rowKv.getRowOffset(), rowKv.getRowLength(), QueryConstants.ARRAY_VALUE_COLUMN_FAMILY, 0, QueryConstants.ARRAY_VALUE_COLUMN_FAMILY.length, QueryConstants.ARRAY_VALUE_COLUMN_QUALIFIER, 0, QueryConstants.ARRAY_VALUE_COLUMN_QUALIFIER.length, HConstants.LATEST_TIMESTAMP, KeyValue.Type.codeToType(rowKv.getTypeByte()), value, 0, value.length));
            return result.size() - 1;
        }

        @Override
        public long getMaxResultSize() {
            return s.getMaxResultSize();
        }

        @Override
        public int getBatch() {
            return s.getBatch();
        }
    };
}

Example 12 with KeyValueColumnExpression

use of org.apache.phoenix.expression.KeyValueColumnExpression in project phoenix by apache.

the class NonAggregateRegionScannerFactory method deserializeArrayPostionalExpressionInfoFromScan.

private Expression[] deserializeArrayPostionalExpressionInfoFromScan(Scan scan, RegionScanner s, Set<KeyValueColumnExpression> arrayKVRefs) {
    byte[] specificArrayIdx = scan.getAttribute(BaseScannerRegionObserver.SPECIFIC_ARRAY_INDEX);
    if (specificArrayIdx == null) {
        return null;
    }
    KeyValueSchema.KeyValueSchemaBuilder builder = new KeyValueSchema.KeyValueSchemaBuilder(0);
    ByteArrayInputStream stream = new ByteArrayInputStream(specificArrayIdx);
    try {
        DataInputStream input = new DataInputStream(stream);
        int arrayKVRefSize = WritableUtils.readVInt(input);
        for (int i = 0; i < arrayKVRefSize; i++) {
            PTable.ImmutableStorageScheme scheme = EncodedColumnsUtil.getImmutableStorageScheme(scan);
            KeyValueColumnExpression kvExp = scheme != PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN ? new SingleCellColumnExpression() : new KeyValueColumnExpression();
            kvExp.readFields(input);
            arrayKVRefs.add(kvExp);
        }
        int arrayKVFuncSize = WritableUtils.readVInt(input);
        Expression[] arrayFuncRefs = new Expression[arrayKVFuncSize];
        for (int i = 0; i < arrayKVFuncSize; i++) {
            ArrayIndexFunction arrayIdxFunc = new ArrayIndexFunction();
            arrayIdxFunc.readFields(input);
            arrayFuncRefs[i] = arrayIdxFunc;
            builder.addField(arrayIdxFunc);
        }
        kvSchema = builder.build();
        kvSchemaBitSet = ValueBitSet.newInstance(kvSchema);
        return arrayFuncRefs;
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        try {
            stream.close();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
}

Example 13 with KeyValueColumnExpression

use of org.apache.phoenix.expression.KeyValueColumnExpression in project phoenix by apache.

the class TestUtil method getSingleSumAggregator.

public static ClientAggregators getSingleSumAggregator(String url, Properties props) throws SQLException {
    try (PhoenixConnection pconn = DriverManager.getConnection(url, props).unwrap(PhoenixConnection.class)) {
        PhoenixStatement statement = new PhoenixStatement(pconn);
        StatementContext context = new StatementContext(statement, null, new Scan(), new SequenceManager(statement));
        AggregationManager aggregationManager = context.getAggregationManager();
        SumAggregateFunction func = new SumAggregateFunction(Arrays.<Expression>asList(new KeyValueColumnExpression(new PLongColumn() {

            @Override
            public PName getName() {
                return SINGLE_COLUMN_NAME;
            }

            @Override
            public PName getFamilyName() {
                return SINGLE_COLUMN_FAMILY_NAME;
            }

            @Override
            public int getPosition() {
                return 0;
            }

            @Override
            public SortOrder getSortOrder() {
                return SortOrder.getDefault();
            }

            @Override
            public Integer getArraySize() {
                return 0;
            }

            @Override
            public byte[] getViewConstant() {
                return null;
            }

            @Override
            public boolean isViewReferenced() {
                return false;
            }

            @Override
            public String getExpressionStr() {
                return null;
            }

            @Override
            public boolean isRowTimestamp() {
                return false;
            }

            @Override
            public boolean isDynamic() {
                return false;
            }

            @Override
            public byte[] getColumnQualifierBytes() {
                return SINGLE_COLUMN_NAME.getBytes();
            }
        })), null);
        aggregationManager.setAggregators(new ClientAggregators(Collections.<SingleAggregateFunction>singletonList(func), 1));
        ClientAggregators aggregators = aggregationManager.getAggregators();
        return aggregators;
    }
}

Example 14 with KeyValueColumnExpression

use of org.apache.phoenix.expression.KeyValueColumnExpression in project phoenix by apache.

the class IndexMaintainer method initCachedState.

/**
     * Init calculated state reading/creating
     */
private void initCachedState() {
    byte[] emptyKvQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(encodingScheme).getFirst();
    dataEmptyKeyValueRef = new ColumnReference(emptyKeyValueCFPtr.copyBytesIfNecessary(), emptyKvQualifier);
    this.allColumns = Sets.newLinkedHashSetWithExpectedSize(indexedExpressions.size() + coveredColumnsMap.size());
    // columns that are required to evaluate all expressions in indexedExpressions (not including columns in data row key)
    this.indexedColumns = Sets.newLinkedHashSetWithExpectedSize(indexedExpressions.size());
    for (Expression expression : indexedExpressions) {
        KeyValueExpressionVisitor visitor = new KeyValueExpressionVisitor() {

            @Override
            public Void visit(KeyValueColumnExpression expression) {
                if (indexedColumns.add(new ColumnReference(expression.getColumnFamily(), expression.getColumnQualifier()))) {
                    indexedColumnTypes.add(expression.getDataType());
                }
                return null;
            }
        };
        expression.accept(visitor);
    }
    allColumns.addAll(indexedColumns);
    for (ColumnReference colRef : coveredColumnsMap.keySet()) {
        if (immutableStorageScheme == ImmutableStorageScheme.ONE_CELL_PER_COLUMN) {
            allColumns.add(colRef);
        } else {
            allColumns.add(new ColumnReference(colRef.getFamily(), QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES));
        }
    }
    int dataPkOffset = (isDataTableSalted ? 1 : 0) + (isMultiTenant ? 1 : 0);
    int nIndexPkColumns = getIndexPkColumnCount();
    dataPkPosition = new int[nIndexPkColumns];
    Arrays.fill(dataPkPosition, EXPRESSION_NOT_PRESENT);
    int numViewConstantColumns = 0;
    BitSet viewConstantColumnBitSet = rowKeyMetaData.getViewConstantColumnBitSet();
    for (int i = dataPkOffset; i < dataRowKeySchema.getFieldCount(); i++) {
        if (!viewConstantColumnBitSet.get(i)) {
            int indexPkPosition = rowKeyMetaData.getIndexPkPosition(i - dataPkOffset);
            this.dataPkPosition[indexPkPosition] = i;
        } else {
            numViewConstantColumns++;
        }
    }
    // Calculate the max number of trailing nulls that we should get rid of after building the index row key.
    // We only get rid of nulls for variable length types, so we have to be careful to consider the type of the
    // index table, not the data type of the data table
    int expressionsPos = indexedExpressions.size();
    int indexPkPos = nIndexPkColumns - numViewConstantColumns - 1;
    while (indexPkPos >= 0) {
        int dataPkPos = dataPkPosition[indexPkPos];
        boolean isDataNullable;
        PDataType dataType;
        if (dataPkPos == EXPRESSION_NOT_PRESENT) {
            isDataNullable = true;
            dataType = indexedExpressions.get(--expressionsPos).getDataType();
        } else {
            Field dataField = dataRowKeySchema.getField(dataPkPos);
            dataType = dataField.getDataType();
            isDataNullable = dataField.isNullable();
        }
        PDataType indexDataType = IndexUtil.getIndexColumnDataType(isDataNullable, dataType);
        if (indexDataType.isFixedWidth()) {
            break;
        }
        indexPkPos--;
    }
    maxTrailingNulls = nIndexPkColumns - indexPkPos - 1;
}

Example 15 with KeyValueColumnExpression

use of org.apache.phoenix.expression.KeyValueColumnExpression in project phoenix by apache.

the class MultiEncodedCQKeyValueComparisonFilter method initFilter.

private void initFilter(Expression expression) {
    cfSet = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
    final BitSet expressionQualifiers = new BitSet(20);
    final Pair<Integer, Integer> range = new Pair<>();
    ExpressionVisitor<Void> visitor = new StatelessTraverseAllExpressionVisitor<Void>() {

        @Override
        public Void visit(KeyValueColumnExpression expression) {
            int qualifier = encodingScheme.decode(expression.getColumnQualifier());
            if (range.getFirst() == null) {
                range.setFirst(qualifier);
                range.setSecond(qualifier);
            } else if (qualifier < range.getFirst()) {
                range.setFirst(qualifier);
            } else if (qualifier > range.getSecond()) {
                range.setSecond(qualifier);
            }
            cfSet.add(expression.getColumnFamily());
            expressionQualifiers.set(qualifier);
            return null;
        }
    };
    expression.accept(visitor);
    // Set min and max qualifiers for columns in the where expression
    whereExpressionMinQualifier = range.getFirst();
    whereExpressionMaxQualifier = range.getSecond();
    int size = whereExpressionMaxQualifier - whereExpressionMinQualifier + 1;
    filteredKeyValues = new FilteredKeyValueHolder(size);
    // Initialize the bitset and mark the qualifiers for columns in where expression
    whereExpressionQualifiers = new BitSet(size);
    for (int i = whereExpressionMinQualifier; i <= whereExpressionMaxQualifier; i++) {
        if (expressionQualifiers.get(i)) {
            whereExpressionQualifiers.set(i - whereExpressionMinQualifier);
        }
    }
    expectedCardinality = whereExpressionQualifiers.cardinality();
}