use of org.apache.phoenix.expression.KeyValueColumnExpression in project phoenix by apache.
the class ProjectionCompiler method compile.
/**
* Builds the projection for the scan
* @param context query context kept between compilation of different query clauses
* @param statement TODO
* @param groupBy compiled GROUP BY clause
* @param targetColumns list of columns, parallel to aliasedNodes, that are being set for an
* UPSERT SELECT statement. Used to coerce expression types to the expected target type.
* @return projector used to access row values during scan
* @throws SQLException
*/
public static RowProjector compile(StatementContext context, SelectStatement statement, GroupBy groupBy, List<? extends PDatum> targetColumns, Expression where) throws SQLException {
List<KeyValueColumnExpression> arrayKVRefs = new ArrayList<KeyValueColumnExpression>();
List<ProjectedColumnExpression> arrayProjectedColumnRefs = new ArrayList<ProjectedColumnExpression>();
List<Expression> arrayKVFuncs = new ArrayList<Expression>();
List<Expression> arrayOldFuncs = new ArrayList<Expression>();
Map<Expression, Integer> arrayExpressionCounts = new HashMap<>();
List<AliasedNode> aliasedNodes = statement.getSelect();
// Setup projected columns in Scan
SelectClauseVisitor selectVisitor = new SelectClauseVisitor(context, groupBy, arrayKVRefs, arrayKVFuncs, arrayExpressionCounts, arrayProjectedColumnRefs, arrayOldFuncs, statement);
List<ExpressionProjector> projectedColumns = new ArrayList<ExpressionProjector>();
ColumnResolver resolver = context.getResolver();
TableRef tableRef = context.getCurrentTable();
PTable table = tableRef.getTable();
boolean resolveColumn = !tableRef.equals(resolver.getTables().get(0));
boolean isWildcard = false;
Scan scan = context.getScan();
int index = 0;
List<Expression> projectedExpressions = Lists.newArrayListWithExpectedSize(aliasedNodes.size());
List<byte[]> projectedFamilies = Lists.newArrayListWithExpectedSize(aliasedNodes.size());
for (AliasedNode aliasedNode : aliasedNodes) {
ParseNode node = aliasedNode.getNode();
// TODO: visitor?
if (node instanceof WildcardParseNode) {
if (statement.isAggregate()) {
ExpressionCompiler.throwNonAggExpressionInAggException(node.toString());
}
if (tableRef == TableRef.EMPTY_TABLE_REF) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_TABLE_SPECIFIED_FOR_WILDCARD_SELECT).build().buildException();
}
isWildcard = true;
if (tableRef.getTable().getType() == PTableType.INDEX && ((WildcardParseNode) node).isRewrite()) {
projectAllIndexColumns(context, tableRef, resolveColumn, projectedExpressions, projectedColumns, targetColumns);
} else {
projectAllTableColumns(context, tableRef, resolveColumn, projectedExpressions, projectedColumns, targetColumns);
}
} else if (node instanceof TableWildcardParseNode) {
TableName tName = ((TableWildcardParseNode) node).getTableName();
TableRef tRef = resolver.resolveTable(tName.getSchemaName(), tName.getTableName());
if (tRef.equals(tableRef)) {
isWildcard = true;
}
if (tRef.getTable().getType() == PTableType.INDEX && ((TableWildcardParseNode) node).isRewrite()) {
projectAllIndexColumns(context, tRef, true, projectedExpressions, projectedColumns, targetColumns);
} else {
projectAllTableColumns(context, tRef, true, projectedExpressions, projectedColumns, targetColumns);
}
} else if (node instanceof FamilyWildcardParseNode) {
if (tableRef == TableRef.EMPTY_TABLE_REF) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_TABLE_SPECIFIED_FOR_WILDCARD_SELECT).build().buildException();
}
// Project everything for SELECT cf.*
String cfName = ((FamilyWildcardParseNode) node).getName();
// Delay projecting to scan, as when any other column in the column family gets
// added to the scan, it overwrites that we want to project the entire column
// family. Instead, we do the projection at the end.
// TODO: consider having a ScanUtil.addColumn and ScanUtil.addFamily to work
// around this, as this code depends on this function being the last place where
// columns are projected (which is currently true, but could change).
projectedFamilies.add(Bytes.toBytes(cfName));
if (tableRef.getTable().getType() == PTableType.INDEX && ((FamilyWildcardParseNode) node).isRewrite()) {
projectIndexColumnFamily(context, cfName, tableRef, resolveColumn, projectedExpressions, projectedColumns);
} else {
projectTableColumnFamily(context, cfName, tableRef, resolveColumn, projectedExpressions, projectedColumns);
}
} else {
Expression expression = node.accept(selectVisitor);
projectedExpressions.add(expression);
expression = coerceIfNecessary(index, targetColumns, expression);
if (node instanceof BindParseNode) {
context.getBindManager().addParamMetaData((BindParseNode) node, expression);
}
if (!node.isStateless()) {
if (!selectVisitor.isAggregate() && statement.isAggregate()) {
ExpressionCompiler.throwNonAggExpressionInAggException(expression.toString());
}
}
String columnAlias = aliasedNode.getAlias() != null ? aliasedNode.getAlias() : SchemaUtil.normalizeIdentifier(aliasedNode.getNode().getAlias());
boolean isCaseSensitive = aliasedNode.getAlias() != null ? aliasedNode.isCaseSensitve() : (columnAlias != null ? SchemaUtil.isCaseSensitive(aliasedNode.getNode().getAlias()) : selectVisitor.isCaseSensitive);
String name = columnAlias == null ? expression.toString() : columnAlias;
projectedColumns.add(new ExpressionProjector(name, tableRef.getTableAlias() == null ? (table.getName() == null ? "" : table.getName().getString()) : tableRef.getTableAlias(), expression, isCaseSensitive));
}
selectVisitor.reset();
index++;
}
for (int i = arrayProjectedColumnRefs.size() - 1; i >= 0; i--) {
Expression expression = arrayProjectedColumnRefs.get(i);
Integer count = arrayExpressionCounts.get(expression);
if (count != 0) {
arrayKVRefs.remove(i);
arrayKVFuncs.remove(i);
arrayOldFuncs.remove(i);
}
}
if (arrayKVFuncs.size() > 0 && arrayKVRefs.size() > 0) {
serailizeArrayIndexInformationAndSetInScan(context, arrayKVFuncs, arrayKVRefs);
KeyValueSchemaBuilder builder = new KeyValueSchemaBuilder(0);
for (Expression expression : arrayKVRefs) {
builder.addField(expression);
}
KeyValueSchema kvSchema = builder.build();
ValueBitSet arrayIndexesBitSet = ValueBitSet.newInstance(kvSchema);
builder = new KeyValueSchemaBuilder(0);
for (Expression expression : arrayKVFuncs) {
builder.addField(expression);
}
KeyValueSchema arrayIndexesSchema = builder.build();
Map<Expression, Expression> replacementMap = new HashMap<>();
for (int i = 0; i < arrayOldFuncs.size(); i++) {
Expression function = arrayKVFuncs.get(i);
replacementMap.put(arrayOldFuncs.get(i), new ArrayIndexExpression(i, function.getDataType(), arrayIndexesBitSet, arrayIndexesSchema));
}
ReplaceArrayFunctionExpressionVisitor visitor = new ReplaceArrayFunctionExpressionVisitor(replacementMap);
for (int i = 0; i < projectedColumns.size(); i++) {
ExpressionProjector projector = projectedColumns.get(i);
projectedColumns.set(i, new ExpressionProjector(projector.getName(), tableRef.getTableAlias() == null ? (table.getName() == null ? "" : table.getName().getString()) : tableRef.getTableAlias(), projector.getExpression().accept(visitor), projector.isCaseSensitive()));
}
}
// TODO make estimatedByteSize more accurate by counting the joined columns.
int estimatedKeySize = table.getRowKeySchema().getEstimatedValueLength();
int estimatedByteSize = 0;
for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
PColumnFamily family = table.getColumnFamily(entry.getKey());
if (entry.getValue() == null) {
for (PColumn column : family.getColumns()) {
Integer maxLength = column.getMaxLength();
int byteSize = column.getDataType().isFixedWidth() ? maxLength == null ? column.getDataType().getByteSize() : maxLength : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE;
estimatedByteSize += SizedUtil.KEY_VALUE_SIZE + estimatedKeySize + byteSize;
}
} else {
for (byte[] cq : entry.getValue()) {
//if (!Bytes.equals(cq, ByteUtil.EMPTY_BYTE_ARRAY) || cq.length > 0) {
PColumn column = family.getPColumnForColumnQualifier(cq);
Integer maxLength = column.getMaxLength();
int byteSize = column.getDataType().isFixedWidth() ? maxLength == null ? column.getDataType().getByteSize() : maxLength : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE;
estimatedByteSize += SizedUtil.KEY_VALUE_SIZE + estimatedKeySize + byteSize;
}
//}
}
}
boolean isProjectEmptyKeyValue = false;
if (isWildcard) {
projectAllColumnFamilies(table, scan);
} else {
isProjectEmptyKeyValue = where == null || LiteralExpression.isTrue(where) || where.requiresFinalEvaluation();
for (byte[] family : projectedFamilies) {
projectColumnFamily(table, scan, family);
}
}
return new RowProjector(projectedColumns, estimatedByteSize, isProjectEmptyKeyValue, resolver.hasUDFs(), isWildcard);
}
use of org.apache.phoenix.expression.KeyValueColumnExpression in project phoenix by apache.
the class IndexMaintainer method readFields.
// Only called by code older than our 4.10 release
@Deprecated
@Override
public void readFields(DataInput input) throws IOException {
int encodedIndexSaltBucketsAndMultiTenant = WritableUtils.readVInt(input);
isMultiTenant = encodedIndexSaltBucketsAndMultiTenant < 0;
nIndexSaltBuckets = Math.abs(encodedIndexSaltBucketsAndMultiTenant) - 1;
int encodedIndexedColumnsAndViewId = WritableUtils.readVInt(input);
boolean hasViewIndexId = encodedIndexedColumnsAndViewId < 0;
if (hasViewIndexId) {
// Fixed length
viewIndexId = new byte[MetaDataUtil.getViewIndexIdDataType().getByteSize()];
input.readFully(viewIndexId);
}
int nIndexedColumns = Math.abs(encodedIndexedColumnsAndViewId) - 1;
indexedColumns = Sets.newLinkedHashSetWithExpectedSize(nIndexedColumns);
for (int i = 0; i < nIndexedColumns; i++) {
byte[] cf = Bytes.readByteArray(input);
byte[] cq = Bytes.readByteArray(input);
indexedColumns.add(new ColumnReference(cf, cq));
}
indexedColumnTypes = Lists.newArrayListWithExpectedSize(nIndexedColumns);
for (int i = 0; i < nIndexedColumns; i++) {
PDataType type = PDataType.values()[WritableUtils.readVInt(input)];
indexedColumnTypes.add(type);
}
int encodedCoveredolumnsAndLocalIndex = WritableUtils.readVInt(input);
isLocalIndex = encodedCoveredolumnsAndLocalIndex < 0;
int nCoveredColumns = Math.abs(encodedCoveredolumnsAndLocalIndex) - 1;
coveredColumnsMap = Maps.newHashMapWithExpectedSize(nCoveredColumns);
for (int i = 0; i < nCoveredColumns; i++) {
byte[] dataTableCf = Bytes.readByteArray(input);
byte[] dataTableCq = Bytes.readByteArray(input);
ColumnReference dataTableRef = new ColumnReference(dataTableCf, dataTableCq);
byte[] indexTableCf = isLocalIndex ? IndexUtil.getLocalIndexColumnFamily(dataTableCf) : dataTableCf;
byte[] indexTableCq = IndexUtil.getIndexColumnName(dataTableCf, dataTableCq);
ColumnReference indexTableRef = new ColumnReference(indexTableCf, indexTableCq);
coveredColumnsMap.put(dataTableRef, indexTableRef);
}
// Hack to serialize whether the index row key is optimizable
int len = WritableUtils.readVInt(input);
if (len < 0) {
rowKeyOrderOptimizable = false;
len *= -1;
} else {
rowKeyOrderOptimizable = true;
}
indexTableName = new byte[len];
input.readFully(indexTableName, 0, len);
dataEmptyKeyValueCF = Bytes.readByteArray(input);
len = WritableUtils.readVInt(input);
//TODO remove this in the next major release
boolean isNewClient = false;
if (len < 0) {
isNewClient = true;
len = Math.abs(len);
}
byte[] emptyKeyValueCF = new byte[len];
input.readFully(emptyKeyValueCF, 0, len);
emptyKeyValueCFPtr = new ImmutableBytesPtr(emptyKeyValueCF);
if (isNewClient) {
int numIndexedExpressions = WritableUtils.readVInt(input);
indexedExpressions = Lists.newArrayListWithExpectedSize(numIndexedExpressions);
for (int i = 0; i < numIndexedExpressions; i++) {
Expression expression = ExpressionType.values()[WritableUtils.readVInt(input)].newInstance();
expression.readFields(input);
indexedExpressions.add(expression);
}
} else {
indexedExpressions = Lists.newArrayListWithExpectedSize(indexedColumns.size());
Iterator<ColumnReference> colReferenceIter = indexedColumns.iterator();
Iterator<PDataType> dataTypeIter = indexedColumnTypes.iterator();
while (colReferenceIter.hasNext()) {
ColumnReference colRef = colReferenceIter.next();
final PDataType dataType = dataTypeIter.next();
indexedExpressions.add(new KeyValueColumnExpression(new PDatum() {
@Override
public boolean isNullable() {
return true;
}
@Override
public SortOrder getSortOrder() {
return SortOrder.getDefault();
}
@Override
public Integer getScale() {
return null;
}
@Override
public Integer getMaxLength() {
return null;
}
@Override
public PDataType getDataType() {
return dataType;
}
}, colRef.getFamily(), colRef.getQualifier()));
}
}
rowKeyMetaData = newRowKeyMetaData();
rowKeyMetaData.readFields(input);
int nDataCFs = WritableUtils.readVInt(input);
// Encode indexWALDisabled in nDataCFs
indexWALDisabled = nDataCFs < 0;
this.nDataCFs = Math.abs(nDataCFs) - 1;
int encodedEstimatedIndexRowKeyBytesAndImmutableRows = WritableUtils.readVInt(input);
this.immutableRows = encodedEstimatedIndexRowKeyBytesAndImmutableRows < 0;
this.estimatedIndexRowKeyBytes = Math.abs(encodedEstimatedIndexRowKeyBytesAndImmutableRows);
// Needed for backward compatibility. Clients older than 4.10 will have non-encoded tables.
this.immutableStorageScheme = ImmutableStorageScheme.ONE_CELL_PER_COLUMN;
this.encodingScheme = QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
initCachedState();
}
use of org.apache.phoenix.expression.KeyValueColumnExpression in project phoenix by apache.
the class PhoenixIndexBuilder method executeAtomicOp.
@Override
public List<Mutation> executeAtomicOp(Increment inc) throws IOException {
byte[] opBytes = inc.getAttribute(ATOMIC_OP_ATTRIB);
if (opBytes == null) {
// Unexpected
return null;
}
inc.setAttribute(ATOMIC_OP_ATTRIB, null);
Put put = null;
Delete delete = null;
// We cannot neither use the time stamp in the Increment to set the Get time range
// nor set the Put/Delete time stamp and have this be atomic as HBase does not
// handle that. Though we disallow using ON DUPLICATE KEY clause when the
// CURRENT_SCN is set, we still may have a time stamp set as of when the table
// was resolved on the client side. We need to ignore this as well due to limitations
// in HBase, but this isn't too bad as the time will be very close the the current
// time anyway.
long ts = HConstants.LATEST_TIMESTAMP;
byte[] rowKey = inc.getRow();
final Get get = new Get(rowKey);
if (isDupKeyIgnore(opBytes)) {
get.setFilter(new FirstKeyOnlyFilter());
Result result = this.env.getRegion().get(get);
return result.isEmpty() ? convertIncrementToPutInSingletonList(inc) : Collections.<Mutation>emptyList();
}
ByteArrayInputStream stream = new ByteArrayInputStream(opBytes);
DataInputStream input = new DataInputStream(stream);
boolean skipFirstOp = input.readBoolean();
short repeat = input.readShort();
final int[] estimatedSizeHolder = { 0 };
List<Pair<PTable, List<Expression>>> operations = Lists.newArrayListWithExpectedSize(3);
while (true) {
ExpressionVisitor<Void> visitor = new StatelessTraverseAllExpressionVisitor<Void>() {
@Override
public Void visit(KeyValueColumnExpression expression) {
get.addColumn(expression.getColumnFamily(), expression.getColumnQualifier());
estimatedSizeHolder[0]++;
return null;
}
};
try {
int nExpressions = WritableUtils.readVInt(input);
List<Expression> expressions = Lists.newArrayListWithExpectedSize(nExpressions);
for (int i = 0; i < nExpressions; i++) {
Expression expression = ExpressionType.values()[WritableUtils.readVInt(input)].newInstance();
expression.readFields(input);
expressions.add(expression);
expression.accept(visitor);
}
PTableProtos.PTable tableProto = PTableProtos.PTable.parseDelimitedFrom(input);
PTable table = PTableImpl.createFromProto(tableProto);
operations.add(new Pair<>(table, expressions));
} catch (EOFException e) {
break;
}
}
int estimatedSize = estimatedSizeHolder[0];
if (get.getFamilyMap().isEmpty()) {
get.setFilter(new FirstKeyOnlyFilter());
}
MultiKeyValueTuple tuple;
List<Cell> flattenedCells = null;
List<Cell> cells = ((HRegion) this.env.getRegion()).get(get, false);
if (cells.isEmpty()) {
if (skipFirstOp) {
if (operations.size() <= 1 && repeat <= 1) {
return convertIncrementToPutInSingletonList(inc);
}
// Skip first operation (if first wasn't ON DUPLICATE KEY IGNORE)
repeat--;
}
// Base current state off of new row
flattenedCells = flattenCells(inc, estimatedSize);
tuple = new MultiKeyValueTuple(flattenedCells);
} else {
// Base current state off of existing row
tuple = new MultiKeyValueTuple(cells);
}
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
for (int opIndex = 0; opIndex < operations.size(); opIndex++) {
Pair<PTable, List<Expression>> operation = operations.get(opIndex);
PTable table = operation.getFirst();
List<Expression> expressions = operation.getSecond();
for (int j = 0; j < repeat; j++) {
// repeater loop
ptr.set(rowKey);
// executed, not when the outer loop is exited. Hence we do it here, at the top of the loop.
if (flattenedCells != null) {
Collections.sort(flattenedCells, KeyValue.COMPARATOR);
}
PRow row = table.newRow(GenericKeyValueBuilder.INSTANCE, ts, ptr, false);
for (int i = 0; i < expressions.size(); i++) {
Expression expression = expressions.get(i);
ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
expression.evaluate(tuple, ptr);
PColumn column = table.getColumns().get(i + 1);
Object value = expression.getDataType().toObject(ptr, column.getSortOrder());
// same type.
if (!column.getDataType().isSizeCompatible(ptr, value, column.getDataType(), expression.getSortOrder(), expression.getMaxLength(), expression.getScale(), column.getMaxLength(), column.getScale())) {
throw new DataExceedsCapacityException(column.getDataType(), column.getMaxLength(), column.getScale());
}
column.getDataType().coerceBytes(ptr, value, expression.getDataType(), expression.getMaxLength(), expression.getScale(), expression.getSortOrder(), column.getMaxLength(), column.getScale(), column.getSortOrder(), table.rowKeyOrderOptimizable());
byte[] bytes = ByteUtil.copyKeyBytesIfNecessary(ptr);
row.setValue(column, bytes);
}
flattenedCells = Lists.newArrayListWithExpectedSize(estimatedSize);
List<Mutation> mutations = row.toRowMutations();
for (Mutation source : mutations) {
flattenCells(source, flattenedCells);
}
tuple.setKeyValues(flattenedCells);
}
// Repeat only applies to first statement
repeat = 1;
}
List<Mutation> mutations = Lists.newArrayListWithExpectedSize(2);
for (int i = 0; i < tuple.size(); i++) {
Cell cell = tuple.getValue(i);
if (Type.codeToType(cell.getTypeByte()) == Type.Put) {
if (put == null) {
put = new Put(rowKey);
transferAttributes(inc, put);
mutations.add(put);
}
put.add(cell);
} else {
if (delete == null) {
delete = new Delete(rowKey);
transferAttributes(inc, delete);
mutations.add(delete);
}
delete.addDeleteMarker(cell);
}
}
return mutations;
}
use of org.apache.phoenix.expression.KeyValueColumnExpression in project phoenix by apache.
the class ColumnRef method newColumnExpression.
public Expression newColumnExpression(boolean schemaNameCaseSensitive, boolean colNameCaseSensitive) throws SQLException {
PTable table = tableRef.getTable();
PColumn column = this.getColumn();
String displayName = tableRef.getColumnDisplayName(this, schemaNameCaseSensitive, colNameCaseSensitive);
if (SchemaUtil.isPKColumn(column)) {
return new RowKeyColumnExpression(column, new RowKeyValueAccessor(table.getPKColumns(), pkSlotPosition), displayName);
}
if (table.getType() == PTableType.PROJECTED || table.getType() == PTableType.SUBQUERY) {
return new ProjectedColumnExpression(column, table, displayName);
}
Expression expression = table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS ? new SingleCellColumnExpression(column, displayName, table.getEncodingScheme()) : new KeyValueColumnExpression(column, displayName);
if (column.getExpressionStr() != null) {
String url = PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + PhoenixRuntime.CONNECTIONLESS;
PhoenixConnection conn = DriverManager.getConnection(url).unwrap(PhoenixConnection.class);
StatementContext context = new StatementContext(new PhoenixStatement(conn));
ExpressionCompiler compiler = new ExpressionCompiler(context);
ParseNode defaultParseNode = new SQLParser(column.getExpressionStr()).parseExpression();
Expression defaultExpression = defaultParseNode.accept(compiler);
if (!ExpressionUtil.isNull(defaultExpression, new ImmutableBytesWritable())) {
return new DefaultValueExpression(Arrays.asList(expression, defaultExpression));
}
}
return expression;
}
use of org.apache.phoenix.expression.KeyValueColumnExpression in project phoenix by apache.
the class IndexUtil method getTupleProjector.
public static TupleProjector getTupleProjector(Scan scan, ColumnReference[] dataColumns) {
if (dataColumns != null && dataColumns.length != 0) {
KeyValueSchema keyValueSchema = deserializeLocalIndexJoinSchemaFromScan(scan);
boolean storeColsInSingleCell = scan.getAttribute(BaseScannerRegionObserver.COLUMNS_STORED_IN_SINGLE_CELL) != null;
QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);
Expression[] colExpressions = storeColsInSingleCell ? new SingleCellColumnExpression[dataColumns.length] : new KeyValueColumnExpression[dataColumns.length];
for (int i = 0; i < dataColumns.length; i++) {
byte[] family = dataColumns[i].getFamily();
byte[] qualifier = dataColumns[i].getQualifier();
Field field = keyValueSchema.getField(i);
Expression dataColumnExpr = storeColsInSingleCell ? new SingleCellColumnExpression(field, family, qualifier, encodingScheme) : new KeyValueColumnExpression(field, family, qualifier);
colExpressions[i] = dataColumnExpr;
}
return new TupleProjector(keyValueSchema, colExpressions);
}
return null;
}
Aggregations