use of org.apache.phoenix.schema.ValueSchema.Field in project phoenix by apache.
the class IndexMaintainer method buildRowKey.
public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKeyPtr, byte[] regionStartKey, byte[] regionEndKey) {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
boolean prependRegionStartKey = isLocalIndex && regionStartKey != null;
boolean isIndexSalted = !isLocalIndex && nIndexSaltBuckets > 0;
int prefixKeyLength = prependRegionStartKey ? (regionStartKey.length != 0 ? regionStartKey.length : regionEndKey.length) : 0;
TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes + (prependRegionStartKey ? prefixKeyLength : 0));
DataOutput output = new DataOutputStream(stream);
try {
// For local indexes, we must prepend the row key with the start region key
if (prependRegionStartKey) {
if (regionStartKey.length == 0) {
output.write(new byte[prefixKeyLength]);
} else {
output.write(regionStartKey);
}
}
if (isIndexSalted) {
// will be set at end to index salt byte
output.write(0);
}
// The dataRowKeySchema includes the salt byte field,
// so we must adjust for that here.
int dataPosOffset = isDataTableSalted ? 1 : 0;
BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
int nIndexedColumns = getIndexPkColumnCount() - getNumViewConstants();
int[][] dataRowKeyLocator = new int[2][nIndexedColumns];
// Skip data table salt byte
int maxRowKeyOffset = rowKeyPtr.getOffset() + rowKeyPtr.getLength();
dataRowKeySchema.iterator(rowKeyPtr, ptr, dataPosOffset);
if (viewIndexId != null) {
output.write(viewIndexId);
}
if (isMultiTenant) {
dataRowKeySchema.next(ptr, dataPosOffset, maxRowKeyOffset);
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(dataPosOffset)));
}
dataPosOffset++;
}
// Write index row key
for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
Boolean hasValue = dataRowKeySchema.next(ptr, i, maxRowKeyOffset);
// same for all rows in this index)
if (!viewConstantColumnBitSet.get(i)) {
int pos = rowKeyMetaData.getIndexPkPosition(i - dataPosOffset);
if (Boolean.TRUE.equals(hasValue)) {
dataRowKeyLocator[0][pos] = ptr.getOffset();
dataRowKeyLocator[1][pos] = ptr.getLength();
} else {
dataRowKeyLocator[0][pos] = 0;
dataRowKeyLocator[1][pos] = 0;
}
}
}
BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
Iterator<Expression> expressionIterator = indexedExpressions.iterator();
for (int i = 0; i < nIndexedColumns; i++) {
PDataType dataColumnType;
boolean isNullable;
SortOrder dataSortOrder;
if (dataPkPosition[i] == EXPRESSION_NOT_PRESENT) {
Expression expression = expressionIterator.next();
dataColumnType = expression.getDataType();
dataSortOrder = expression.getSortOrder();
isNullable = expression.isNullable();
expression.evaluate(new ValueGetterTuple(valueGetter), ptr);
} else {
Field field = dataRowKeySchema.getField(dataPkPosition[i]);
dataColumnType = field.getDataType();
ptr.set(rowKeyPtr.get(), dataRowKeyLocator[0][i], dataRowKeyLocator[1][i]);
dataSortOrder = field.getSortOrder();
isNullable = field.isNullable();
}
boolean isDataColumnInverted = dataSortOrder != SortOrder.ASC;
PDataType indexColumnType = IndexUtil.getIndexColumnDataType(isNullable, dataColumnType);
boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType);
boolean isIndexColumnDesc = descIndexColumnBitSet.get(i);
if (isBytesComparable && isDataColumnInverted == isIndexColumnDesc) {
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
} else {
if (!isBytesComparable) {
indexColumnType.coerceBytes(ptr, dataColumnType, dataSortOrder, SortOrder.getDefault());
}
if (isDataColumnInverted != isIndexColumnDesc) {
writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
} else {
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
}
}
if (!indexColumnType.isFixedWidth()) {
output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, isIndexColumnDesc ? SortOrder.DESC : SortOrder.ASC));
}
}
int length = stream.size();
int minLength = length - maxTrailingNulls;
byte[] indexRowKey = stream.getBuffer();
// Remove trailing nulls
while (length > minLength && indexRowKey[length - 1] == QueryConstants.SEPARATOR_BYTE) {
length--;
}
if (isIndexSalted) {
// Set salt byte
byte saltByte = SaltingUtil.getSaltingByte(indexRowKey, SaltingUtil.NUM_SALTING_BYTES, length - SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
indexRowKey[0] = saltByte;
}
return indexRowKey.length == length ? indexRowKey : Arrays.copyOf(indexRowKey, length);
} catch (IOException e) {
// Impossible
throw new RuntimeException(e);
} finally {
try {
stream.close();
} catch (IOException e) {
// Impossible
throw new RuntimeException(e);
}
}
}
use of org.apache.phoenix.schema.ValueSchema.Field in project phoenix by apache.
the class IndexMaintainer method buildDataRowKey.
/*
* Build the data row key from the index row key
*/
public byte[] buildDataRowKey(ImmutableBytesWritable indexRowKeyPtr, byte[][] viewConstants) {
RowKeySchema indexRowKeySchema = getIndexRowKeySchema();
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes);
DataOutput output = new DataOutputStream(stream);
// Increment dataPosOffset until all have been written
int dataPosOffset = 0;
int viewConstantsIndex = 0;
try {
int indexPosOffset = !isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0;
int maxRowKeyOffset = indexRowKeyPtr.getOffset() + indexRowKeyPtr.getLength();
indexRowKeySchema.iterator(indexRowKeyPtr, ptr, indexPosOffset);
if (isDataTableSalted) {
dataPosOffset++;
// will be set at end to salt byte
output.write(0);
}
if (viewIndexId != null) {
indexRowKeySchema.next(ptr, indexPosOffset++, maxRowKeyOffset);
}
if (isMultiTenant) {
indexRowKeySchema.next(ptr, indexPosOffset, maxRowKeyOffset);
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(dataPosOffset)));
}
indexPosOffset++;
dataPosOffset++;
}
indexPosOffset = (!isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0) + (isMultiTenant ? 1 : 0) + (viewIndexId == null ? 0 : 1);
BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
// same for all rows in this index)
if (viewConstantColumnBitSet.get(i)) {
output.write(viewConstants[viewConstantsIndex++]);
} else {
int pos = rowKeyMetaData.getIndexPkPosition(i - dataPosOffset);
Boolean hasValue = indexRowKeySchema.iterator(indexRowKeyPtr, ptr, pos + indexPosOffset + 1);
if (Boolean.TRUE.equals(hasValue)) {
// Write data row key value taking into account coercion and inversion
// if necessary
Field dataField = dataRowKeySchema.getField(i);
Field indexField = indexRowKeySchema.getField(pos + indexPosOffset);
PDataType indexColumnType = indexField.getDataType();
PDataType dataColumnType = dataField.getDataType();
SortOrder dataSortOrder = dataField.getSortOrder();
SortOrder indexSortOrder = indexField.getSortOrder();
boolean isDataColumnInverted = dataSortOrder != SortOrder.ASC;
boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType);
if (isBytesComparable && isDataColumnInverted == descIndexColumnBitSet.get(pos)) {
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
} else {
if (!isBytesComparable) {
dataColumnType.coerceBytes(ptr, indexColumnType, indexSortOrder, SortOrder.getDefault());
}
if (descIndexColumnBitSet.get(pos) != isDataColumnInverted) {
writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
} else {
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
}
}
}
}
// Write separator byte if variable length unless it's the last field in the schema
// (but we still need to write it if it's DESC to ensure sort order is correct).
byte sepByte = SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(i));
if (!dataRowKeySchema.getField(i).getDataType().isFixedWidth() && (((i + 1) != dataRowKeySchema.getFieldCount()) || sepByte == QueryConstants.DESC_SEPARATOR_BYTE)) {
output.writeByte(sepByte);
}
}
int length = stream.size();
int minLength = length - maxTrailingNulls;
byte[] dataRowKey = stream.getBuffer();
// Remove trailing nulls
while (length > minLength && dataRowKey[length - 1] == QueryConstants.SEPARATOR_BYTE) {
length--;
}
// there to maintain compatibility between an old client and a new server.
if (isDataTableSalted) {
// Set salt byte
byte saltByte = SaltingUtil.getSaltingByte(dataRowKey, SaltingUtil.NUM_SALTING_BYTES, length - SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
dataRowKey[0] = saltByte;
}
return dataRowKey.length == length ? dataRowKey : Arrays.copyOf(dataRowKey, length);
} catch (IOException e) {
// Impossible
throw new RuntimeException(e);
} finally {
try {
stream.close();
} catch (IOException e) {
// Impossible
throw new RuntimeException(e);
}
}
}
use of org.apache.phoenix.schema.ValueSchema.Field in project phoenix by apache.
the class ScanUtil method getTenantIdBytes.
public static byte[] getTenantIdBytes(RowKeySchema schema, boolean isSalted, PName tenantId, boolean isSharedIndex) throws SQLException {
int pkPos = (isSalted ? 1 : 0) + (isSharedIndex ? 1 : 0);
Field field = schema.getField(pkPos);
PDataType dataType = field.getDataType();
byte[] convertedValue;
try {
Object value = dataType.toObject(tenantId.getString());
convertedValue = dataType.toBytes(value);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(convertedValue);
dataType.pad(ptr, field.getMaxLength(), field.getSortOrder());
convertedValue = ByteUtil.copyKeyBytesIfNecessary(ptr);
} catch (IllegalDataException ex) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.TENANTID_IS_OF_WRONG_TYPE).build().buildException();
}
return convertedValue;
}
use of org.apache.phoenix.schema.ValueSchema.Field in project phoenix by apache.
the class IndexUtil method getTupleProjector.
public static TupleProjector getTupleProjector(Scan scan, ColumnReference[] dataColumns) {
if (dataColumns != null && dataColumns.length != 0) {
KeyValueSchema keyValueSchema = deserializeLocalIndexJoinSchemaFromScan(scan);
boolean storeColsInSingleCell = scan.getAttribute(BaseScannerRegionObserver.COLUMNS_STORED_IN_SINGLE_CELL) != null;
QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);
Expression[] colExpressions = storeColsInSingleCell ? new SingleCellColumnExpression[dataColumns.length] : new KeyValueColumnExpression[dataColumns.length];
for (int i = 0; i < dataColumns.length; i++) {
byte[] family = dataColumns[i].getFamily();
byte[] qualifier = dataColumns[i].getQualifier();
Field field = keyValueSchema.getField(i);
Expression dataColumnExpr = storeColsInSingleCell ? new SingleCellColumnExpression(field, family, qualifier, encodingScheme) : new KeyValueColumnExpression(field, family, qualifier);
colExpressions[i] = dataColumnExpr;
}
return new TupleProjector(keyValueSchema, colExpressions);
}
return null;
}
use of org.apache.phoenix.schema.ValueSchema.Field in project phoenix by apache.
the class ScanUtil method setKey.
public static int setKey(RowKeySchema schema, List<List<KeyRange>> slots, int[] slotSpan, int[] position, Bound bound, byte[] key, int byteOffset, int slotStartIndex, int slotEndIndex, int schemaStartIndex) {
int offset = byteOffset;
boolean lastInclusiveUpperSingleKey = false;
boolean anyInclusiveUpperRangeKey = false;
boolean lastUnboundUpper = false;
// The index used for slots should be incremented by 1,
// but the index for the field it represents in the schema
// should be incremented by 1 + value in the current slotSpan index
// slotSpan stores the number of columns beyond one that the range spans
Field field = null;
int i = slotStartIndex, fieldIndex = ScanUtil.getRowKeyPosition(slotSpan, slotStartIndex);
for (i = slotStartIndex; i < slotEndIndex; i++) {
// Build up the key by appending the bound of each key range
// from the current position of each slot.
KeyRange range = slots.get(i).get(position[i]);
// Use last slot in a multi-span column to determine if fixed width
field = schema.getField(fieldIndex + slotSpan[i]);
boolean isFixedWidth = field.getDataType().isFixedWidth();
/*
* If the current slot is unbound then stop if:
* 1) setting the upper bound. There's no value in
* continuing because nothing will be filtered.
* 2) setting the lower bound when the type is fixed length
* for the same reason. However, if the type is variable width
* continue building the key because null values will be filtered
* since our separator byte will be appended and incremented.
* 3) if the range includes everything as we cannot add any more useful
* information to the key after that.
*/
lastUnboundUpper = false;
if (range.isUnbound(bound) && (bound == Bound.UPPER || isFixedWidth || range == KeyRange.EVERYTHING_RANGE)) {
lastUnboundUpper = (bound == Bound.UPPER);
break;
}
byte[] bytes = range.getRange(bound);
System.arraycopy(bytes, 0, key, offset, bytes.length);
offset += bytes.length;
/*
* We must add a terminator to a variable length key even for the last PK column if
* the lower key is non inclusive or the upper key is inclusive. Otherwise, we'd be
* incrementing the key value itself, and thus bumping it up too much.
*/
boolean inclusiveUpper = range.isUpperInclusive() && bound == Bound.UPPER;
boolean exclusiveLower = !range.isLowerInclusive() && bound == Bound.LOWER && range != KeyRange.EVERYTHING_RANGE;
boolean exclusiveUpper = !range.isUpperInclusive() && bound == Bound.UPPER;
// If we are setting the upper bound of using inclusive single key, we remember
// to increment the key if we exit the loop after this iteration.
//
// We remember to increment the last slot if we are setting the upper bound with an
// inclusive range key.
//
// We cannot combine the two flags together in case for single-inclusive key followed
// by the range-exclusive key. In that case, we do not need to increment the end at the
// end. But if we combine the two flag, the single inclusive key in the middle of the
// key slots would cause the flag to become true.
lastInclusiveUpperSingleKey = range.isSingleKey() && inclusiveUpper;
anyInclusiveUpperRangeKey |= !range.isSingleKey() && inclusiveUpper;
// A null or empty byte array is always represented as a zero byte
byte sepByte = SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), bytes.length == 0, field);
if (!isFixedWidth && (sepByte == QueryConstants.DESC_SEPARATOR_BYTE || (!exclusiveUpper && (fieldIndex < schema.getMaxFields() || inclusiveUpper || exclusiveLower)))) {
key[offset++] = sepByte;
// Set lastInclusiveUpperSingleKey back to false if this is the last pk column
// as we don't want to increment the null byte in this case
lastInclusiveUpperSingleKey &= i < schema.getMaxFields() - 1;
}
if (exclusiveUpper) {
// would match k1 = 2, k2 = 3 which is wrong.
break;
}
// only after the last key part.
if (exclusiveLower) {
if (!ByteUtil.nextKey(key, offset)) {
// have an end key specified.
return -byteOffset;
}
// just added and incremented.
if (!isFixedWidth && bytes.length == 0 && SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), false, field) == QueryConstants.DESC_SEPARATOR_BYTE) {
key[offset++] = QueryConstants.DESC_SEPARATOR_BYTE;
}
}
fieldIndex += slotSpan[i] + 1;
}
if (lastInclusiveUpperSingleKey || anyInclusiveUpperRangeKey || lastUnboundUpper) {
if (!ByteUtil.nextKey(key, offset)) {
// have an end key specified.
return -byteOffset;
}
}
// byte.
if (bound == Bound.LOWER) {
while (--i >= schemaStartIndex && offset > byteOffset && !(field = schema.getField(--fieldIndex)).getDataType().isFixedWidth() && field.getSortOrder() == SortOrder.ASC && key[offset - 1] == QueryConstants.SEPARATOR_BYTE) {
offset--;
fieldIndex -= slotSpan[i];
}
}
return offset - byteOffset;
}
Aggregations