use of org.apache.phoenix.schema.types.PDataType in project phoenix by apache.
the class IndexMaintainer method toProto.
public static ServerCachingProtos.IndexMaintainer toProto(IndexMaintainer maintainer) throws IOException {
ServerCachingProtos.IndexMaintainer.Builder builder = ServerCachingProtos.IndexMaintainer.newBuilder();
builder.setSaltBuckets(maintainer.nIndexSaltBuckets);
builder.setIsMultiTenant(maintainer.isMultiTenant);
if (maintainer.viewIndexId != null) {
builder.setViewIndexId(ByteStringer.wrap(maintainer.viewIndexId));
}
for (ColumnReference colRef : maintainer.indexedColumns) {
ServerCachingProtos.ColumnReference.Builder cRefBuilder = ServerCachingProtos.ColumnReference.newBuilder();
cRefBuilder.setFamily(ByteStringer.wrap(colRef.getFamily()));
cRefBuilder.setQualifier(ByteStringer.wrap(colRef.getQualifier()));
builder.addIndexedColumns(cRefBuilder.build());
}
for (PDataType dataType : maintainer.indexedColumnTypes) {
builder.addIndexedColumnTypeOrdinal(dataType.ordinal());
}
for (Entry<ColumnReference, ColumnReference> e : maintainer.coveredColumnsMap.entrySet()) {
ServerCachingProtos.ColumnReference.Builder cRefBuilder = ServerCachingProtos.ColumnReference.newBuilder();
ColumnReference dataTableColRef = e.getKey();
cRefBuilder.setFamily(ByteStringer.wrap(dataTableColRef.getFamily()));
cRefBuilder.setQualifier(ByteStringer.wrap(dataTableColRef.getQualifier()));
builder.addDataTableColRefForCoveredColumns(cRefBuilder.build());
if (maintainer.encodingScheme != NON_ENCODED_QUALIFIERS) {
// We need to serialize the colRefs of index tables only in case of encoded column names.
ColumnReference indexTableColRef = e.getValue();
cRefBuilder = ServerCachingProtos.ColumnReference.newBuilder();
cRefBuilder.setFamily(ByteStringer.wrap(indexTableColRef.getFamily()));
cRefBuilder.setQualifier(ByteStringer.wrap(indexTableColRef.getQualifier()));
builder.addIndexTableColRefForCoveredColumns(cRefBuilder.build());
}
}
builder.setIsLocalIndex(maintainer.isLocalIndex);
builder.setIndexTableName(ByteStringer.wrap(maintainer.indexTableName));
builder.setRowKeyOrderOptimizable(maintainer.rowKeyOrderOptimizable);
builder.setDataTableEmptyKeyValueColFamily(ByteStringer.wrap(maintainer.dataEmptyKeyValueCF));
ServerCachingProtos.ImmutableBytesWritable.Builder ibwBuilder = ServerCachingProtos.ImmutableBytesWritable.newBuilder();
ibwBuilder.setByteArray(ByteStringer.wrap(maintainer.emptyKeyValueCFPtr.get()));
ibwBuilder.setLength(maintainer.emptyKeyValueCFPtr.getLength());
ibwBuilder.setOffset(maintainer.emptyKeyValueCFPtr.getOffset());
builder.setEmptyKeyValueColFamily(ibwBuilder.build());
try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) {
DataOutput output = new DataOutputStream(stream);
for (Expression expression : maintainer.indexedExpressions) {
WritableUtils.writeVInt(output, ExpressionType.valueOf(expression).ordinal());
expression.write(output);
}
builder.setIndexedExpressions(ByteStringer.wrap(stream.toByteArray()));
}
try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) {
DataOutput output = new DataOutputStream(stream);
maintainer.rowKeyMetaData.write(output);
builder.setRowKeyMetadata(ByteStringer.wrap(stream.toByteArray()));
}
builder.setNumDataTableColFamilies(maintainer.nDataCFs);
builder.setIndexWalDisabled(maintainer.indexWALDisabled);
builder.setIndexRowKeyByteSize(maintainer.estimatedIndexRowKeyBytes);
builder.setImmutable(maintainer.immutableRows);
for (Pair<String, String> p : maintainer.indexedColumnsInfo) {
ServerCachingProtos.ColumnInfo.Builder ciBuilder = ServerCachingProtos.ColumnInfo.newBuilder();
if (p.getFirst() != null) {
ciBuilder.setFamilyName(p.getFirst());
}
ciBuilder.setColumnName(p.getSecond());
builder.addIndexedColumnInfo(ciBuilder.build());
}
builder.setEncodingScheme(maintainer.encodingScheme.getSerializedMetadataValue());
builder.setImmutableStorageScheme(maintainer.immutableStorageScheme.getSerializedMetadataValue());
return builder.build();
}
use of org.apache.phoenix.schema.types.PDataType in project phoenix by apache.
the class IndexMaintainer method readFields.
// Only called by code older than our 4.10 release
@Deprecated
@Override
public void readFields(DataInput input) throws IOException {
int encodedIndexSaltBucketsAndMultiTenant = WritableUtils.readVInt(input);
isMultiTenant = encodedIndexSaltBucketsAndMultiTenant < 0;
nIndexSaltBuckets = Math.abs(encodedIndexSaltBucketsAndMultiTenant) - 1;
int encodedIndexedColumnsAndViewId = WritableUtils.readVInt(input);
boolean hasViewIndexId = encodedIndexedColumnsAndViewId < 0;
if (hasViewIndexId) {
// Fixed length
viewIndexId = new byte[MetaDataUtil.getViewIndexIdDataType().getByteSize()];
input.readFully(viewIndexId);
}
int nIndexedColumns = Math.abs(encodedIndexedColumnsAndViewId) - 1;
indexedColumns = Sets.newLinkedHashSetWithExpectedSize(nIndexedColumns);
for (int i = 0; i < nIndexedColumns; i++) {
byte[] cf = Bytes.readByteArray(input);
byte[] cq = Bytes.readByteArray(input);
indexedColumns.add(new ColumnReference(cf, cq));
}
indexedColumnTypes = Lists.newArrayListWithExpectedSize(nIndexedColumns);
for (int i = 0; i < nIndexedColumns; i++) {
PDataType type = PDataType.values()[WritableUtils.readVInt(input)];
indexedColumnTypes.add(type);
}
int encodedCoveredolumnsAndLocalIndex = WritableUtils.readVInt(input);
isLocalIndex = encodedCoveredolumnsAndLocalIndex < 0;
int nCoveredColumns = Math.abs(encodedCoveredolumnsAndLocalIndex) - 1;
coveredColumnsMap = Maps.newHashMapWithExpectedSize(nCoveredColumns);
for (int i = 0; i < nCoveredColumns; i++) {
byte[] dataTableCf = Bytes.readByteArray(input);
byte[] dataTableCq = Bytes.readByteArray(input);
ColumnReference dataTableRef = new ColumnReference(dataTableCf, dataTableCq);
byte[] indexTableCf = isLocalIndex ? IndexUtil.getLocalIndexColumnFamily(dataTableCf) : dataTableCf;
byte[] indexTableCq = IndexUtil.getIndexColumnName(dataTableCf, dataTableCq);
ColumnReference indexTableRef = new ColumnReference(indexTableCf, indexTableCq);
coveredColumnsMap.put(dataTableRef, indexTableRef);
}
// Hack to serialize whether the index row key is optimizable
int len = WritableUtils.readVInt(input);
if (len < 0) {
rowKeyOrderOptimizable = false;
len *= -1;
} else {
rowKeyOrderOptimizable = true;
}
indexTableName = new byte[len];
input.readFully(indexTableName, 0, len);
dataEmptyKeyValueCF = Bytes.readByteArray(input);
len = WritableUtils.readVInt(input);
//TODO remove this in the next major release
boolean isNewClient = false;
if (len < 0) {
isNewClient = true;
len = Math.abs(len);
}
byte[] emptyKeyValueCF = new byte[len];
input.readFully(emptyKeyValueCF, 0, len);
emptyKeyValueCFPtr = new ImmutableBytesPtr(emptyKeyValueCF);
if (isNewClient) {
int numIndexedExpressions = WritableUtils.readVInt(input);
indexedExpressions = Lists.newArrayListWithExpectedSize(numIndexedExpressions);
for (int i = 0; i < numIndexedExpressions; i++) {
Expression expression = ExpressionType.values()[WritableUtils.readVInt(input)].newInstance();
expression.readFields(input);
indexedExpressions.add(expression);
}
} else {
indexedExpressions = Lists.newArrayListWithExpectedSize(indexedColumns.size());
Iterator<ColumnReference> colReferenceIter = indexedColumns.iterator();
Iterator<PDataType> dataTypeIter = indexedColumnTypes.iterator();
while (colReferenceIter.hasNext()) {
ColumnReference colRef = colReferenceIter.next();
final PDataType dataType = dataTypeIter.next();
indexedExpressions.add(new KeyValueColumnExpression(new PDatum() {
@Override
public boolean isNullable() {
return true;
}
@Override
public SortOrder getSortOrder() {
return SortOrder.getDefault();
}
@Override
public Integer getScale() {
return null;
}
@Override
public Integer getMaxLength() {
return null;
}
@Override
public PDataType getDataType() {
return dataType;
}
}, colRef.getFamily(), colRef.getQualifier()));
}
}
rowKeyMetaData = newRowKeyMetaData();
rowKeyMetaData.readFields(input);
int nDataCFs = WritableUtils.readVInt(input);
// Encode indexWALDisabled in nDataCFs
indexWALDisabled = nDataCFs < 0;
this.nDataCFs = Math.abs(nDataCFs) - 1;
int encodedEstimatedIndexRowKeyBytesAndImmutableRows = WritableUtils.readVInt(input);
this.immutableRows = encodedEstimatedIndexRowKeyBytesAndImmutableRows < 0;
this.estimatedIndexRowKeyBytes = Math.abs(encodedEstimatedIndexRowKeyBytesAndImmutableRows);
// Needed for backward compatibility. Clients older than 4.10 will have non-encoded tables.
this.immutableStorageScheme = ImmutableStorageScheme.ONE_CELL_PER_COLUMN;
this.encodingScheme = QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
initCachedState();
}
use of org.apache.phoenix.schema.types.PDataType in project phoenix by apache.
the class IndexMaintainer method buildRowKey.
public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKeyPtr, byte[] regionStartKey, byte[] regionEndKey) {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
boolean prependRegionStartKey = isLocalIndex && regionStartKey != null;
boolean isIndexSalted = !isLocalIndex && nIndexSaltBuckets > 0;
int prefixKeyLength = prependRegionStartKey ? (regionStartKey.length != 0 ? regionStartKey.length : regionEndKey.length) : 0;
TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes + (prependRegionStartKey ? prefixKeyLength : 0));
DataOutput output = new DataOutputStream(stream);
try {
// For local indexes, we must prepend the row key with the start region key
if (prependRegionStartKey) {
if (regionStartKey.length == 0) {
output.write(new byte[prefixKeyLength]);
} else {
output.write(regionStartKey);
}
}
if (isIndexSalted) {
// will be set at end to index salt byte
output.write(0);
}
// The dataRowKeySchema includes the salt byte field,
// so we must adjust for that here.
int dataPosOffset = isDataTableSalted ? 1 : 0;
BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
int nIndexedColumns = getIndexPkColumnCount() - getNumViewConstants();
int[][] dataRowKeyLocator = new int[2][nIndexedColumns];
// Skip data table salt byte
int maxRowKeyOffset = rowKeyPtr.getOffset() + rowKeyPtr.getLength();
dataRowKeySchema.iterator(rowKeyPtr, ptr, dataPosOffset);
if (viewIndexId != null) {
output.write(viewIndexId);
}
if (isMultiTenant) {
dataRowKeySchema.next(ptr, dataPosOffset, maxRowKeyOffset);
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(dataPosOffset)));
}
dataPosOffset++;
}
// Write index row key
for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
Boolean hasValue = dataRowKeySchema.next(ptr, i, maxRowKeyOffset);
// same for all rows in this index)
if (!viewConstantColumnBitSet.get(i)) {
int pos = rowKeyMetaData.getIndexPkPosition(i - dataPosOffset);
if (Boolean.TRUE.equals(hasValue)) {
dataRowKeyLocator[0][pos] = ptr.getOffset();
dataRowKeyLocator[1][pos] = ptr.getLength();
} else {
dataRowKeyLocator[0][pos] = 0;
dataRowKeyLocator[1][pos] = 0;
}
}
}
BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
Iterator<Expression> expressionIterator = indexedExpressions.iterator();
for (int i = 0; i < nIndexedColumns; i++) {
PDataType dataColumnType;
boolean isNullable;
SortOrder dataSortOrder;
if (dataPkPosition[i] == EXPRESSION_NOT_PRESENT) {
Expression expression = expressionIterator.next();
dataColumnType = expression.getDataType();
dataSortOrder = expression.getSortOrder();
isNullable = expression.isNullable();
expression.evaluate(new ValueGetterTuple(valueGetter), ptr);
} else {
Field field = dataRowKeySchema.getField(dataPkPosition[i]);
dataColumnType = field.getDataType();
ptr.set(rowKeyPtr.get(), dataRowKeyLocator[0][i], dataRowKeyLocator[1][i]);
dataSortOrder = field.getSortOrder();
isNullable = field.isNullable();
}
boolean isDataColumnInverted = dataSortOrder != SortOrder.ASC;
PDataType indexColumnType = IndexUtil.getIndexColumnDataType(isNullable, dataColumnType);
boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType);
boolean isIndexColumnDesc = descIndexColumnBitSet.get(i);
if (isBytesComparable && isDataColumnInverted == isIndexColumnDesc) {
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
} else {
if (!isBytesComparable) {
indexColumnType.coerceBytes(ptr, dataColumnType, dataSortOrder, SortOrder.getDefault());
}
if (isDataColumnInverted != isIndexColumnDesc) {
writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
} else {
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
}
}
if (!indexColumnType.isFixedWidth()) {
output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, isIndexColumnDesc ? SortOrder.DESC : SortOrder.ASC));
}
}
int length = stream.size();
int minLength = length - maxTrailingNulls;
byte[] indexRowKey = stream.getBuffer();
// Remove trailing nulls
while (length > minLength && indexRowKey[length - 1] == QueryConstants.SEPARATOR_BYTE) {
length--;
}
if (isIndexSalted) {
// Set salt byte
byte saltByte = SaltingUtil.getSaltingByte(indexRowKey, SaltingUtil.NUM_SALTING_BYTES, length - SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
indexRowKey[0] = saltByte;
}
return indexRowKey.length == length ? indexRowKey : Arrays.copyOf(indexRowKey, length);
} catch (IOException e) {
// Impossible
throw new RuntimeException(e);
} finally {
try {
stream.close();
} catch (IOException e) {
// Impossible
throw new RuntimeException(e);
}
}
}
use of org.apache.phoenix.schema.types.PDataType in project phoenix by apache.
the class IndexMaintainer method write.
// Only called by code older than our 4.10 release
@Deprecated
@Override
public void write(DataOutput output) throws IOException {
// Encode nIndexSaltBuckets and isMultiTenant together
WritableUtils.writeVInt(output, (nIndexSaltBuckets + 1) * (isMultiTenant ? -1 : 1));
// Encode indexedColumns.size() and whether or not there's a viewIndexId
WritableUtils.writeVInt(output, (indexedColumns.size() + 1) * (viewIndexId != null ? -1 : 1));
if (viewIndexId != null) {
output.write(viewIndexId);
}
for (ColumnReference ref : indexedColumns) {
Bytes.writeByteArray(output, ref.getFamily());
Bytes.writeByteArray(output, ref.getQualifier());
}
//TODO remove indexedColumnTypes in the next major release
for (int i = 0; i < indexedColumnTypes.size(); i++) {
PDataType type = indexedColumnTypes.get(i);
WritableUtils.writeVInt(output, type.ordinal());
}
// Encode coveredColumns.size() and whether or not this is a local index
WritableUtils.writeVInt(output, (coveredColumnsMap.size() + 1) * (isLocalIndex ? -1 : 1));
for (ColumnReference ref : coveredColumnsMap.keySet()) {
Bytes.writeByteArray(output, ref.getFamily());
Bytes.writeByteArray(output, ref.getQualifier());
}
// TODO: remove when rowKeyOrderOptimizable hack no longer needed
WritableUtils.writeVInt(output, indexTableName.length * (rowKeyOrderOptimizable ? 1 : -1));
output.write(indexTableName, 0, indexTableName.length);
Bytes.writeByteArray(output, dataEmptyKeyValueCF);
// TODO in order to maintain b/w compatibility encode emptyKeyValueCFPtr.getLength() as a negative value (so we can distinguish between new and old clients)
// when indexedColumnTypes is removed, remove this
WritableUtils.writeVInt(output, -emptyKeyValueCFPtr.getLength());
output.write(emptyKeyValueCFPtr.get(), emptyKeyValueCFPtr.getOffset(), emptyKeyValueCFPtr.getLength());
WritableUtils.writeVInt(output, indexedExpressions.size());
for (Expression expression : indexedExpressions) {
WritableUtils.writeVInt(output, ExpressionType.valueOf(expression).ordinal());
expression.write(output);
}
rowKeyMetaData.write(output);
// Encode indexWALDisabled in nDataCFs
WritableUtils.writeVInt(output, (nDataCFs + 1) * (indexWALDisabled ? -1 : 1));
// Encode estimatedIndexRowKeyBytes and immutableRows together.
WritableUtils.writeVInt(output, estimatedIndexRowKeyBytes * (immutableRows ? -1 : 1));
}
use of org.apache.phoenix.schema.types.PDataType in project phoenix by apache.
the class IndexMaintainer method buildDataRowKey.
/*
* Build the data row key from the index row key
*/
public byte[] buildDataRowKey(ImmutableBytesWritable indexRowKeyPtr, byte[][] viewConstants) {
RowKeySchema indexRowKeySchema = getIndexRowKeySchema();
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes);
DataOutput output = new DataOutputStream(stream);
// Increment dataPosOffset until all have been written
int dataPosOffset = 0;
int viewConstantsIndex = 0;
try {
int indexPosOffset = !isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0;
int maxRowKeyOffset = indexRowKeyPtr.getOffset() + indexRowKeyPtr.getLength();
indexRowKeySchema.iterator(indexRowKeyPtr, ptr, indexPosOffset);
if (isDataTableSalted) {
dataPosOffset++;
// will be set at end to salt byte
output.write(0);
}
if (viewIndexId != null) {
indexRowKeySchema.next(ptr, indexPosOffset++, maxRowKeyOffset);
}
if (isMultiTenant) {
indexRowKeySchema.next(ptr, indexPosOffset, maxRowKeyOffset);
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(dataPosOffset)));
}
indexPosOffset++;
dataPosOffset++;
}
indexPosOffset = (!isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0) + (isMultiTenant ? 1 : 0) + (viewIndexId == null ? 0 : 1);
BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
// same for all rows in this index)
if (viewConstantColumnBitSet.get(i)) {
output.write(viewConstants[viewConstantsIndex++]);
} else {
int pos = rowKeyMetaData.getIndexPkPosition(i - dataPosOffset);
Boolean hasValue = indexRowKeySchema.iterator(indexRowKeyPtr, ptr, pos + indexPosOffset + 1);
if (Boolean.TRUE.equals(hasValue)) {
// Write data row key value taking into account coercion and inversion
// if necessary
Field dataField = dataRowKeySchema.getField(i);
Field indexField = indexRowKeySchema.getField(pos + indexPosOffset);
PDataType indexColumnType = indexField.getDataType();
PDataType dataColumnType = dataField.getDataType();
SortOrder dataSortOrder = dataField.getSortOrder();
SortOrder indexSortOrder = indexField.getSortOrder();
boolean isDataColumnInverted = dataSortOrder != SortOrder.ASC;
boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType);
if (isBytesComparable && isDataColumnInverted == descIndexColumnBitSet.get(pos)) {
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
} else {
if (!isBytesComparable) {
dataColumnType.coerceBytes(ptr, indexColumnType, indexSortOrder, SortOrder.getDefault());
}
if (descIndexColumnBitSet.get(pos) != isDataColumnInverted) {
writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
} else {
output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
}
}
}
}
// Write separator byte if variable length unless it's the last field in the schema
// (but we still need to write it if it's DESC to ensure sort order is correct).
byte sepByte = SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(i));
if (!dataRowKeySchema.getField(i).getDataType().isFixedWidth() && (((i + 1) != dataRowKeySchema.getFieldCount()) || sepByte == QueryConstants.DESC_SEPARATOR_BYTE)) {
output.writeByte(sepByte);
}
}
int length = stream.size();
int minLength = length - maxTrailingNulls;
byte[] dataRowKey = stream.getBuffer();
// Remove trailing nulls
while (length > minLength && dataRowKey[length - 1] == QueryConstants.SEPARATOR_BYTE) {
length--;
}
// there to maintain compatibility between an old client and a new server.
if (isDataTableSalted) {
// Set salt byte
byte saltByte = SaltingUtil.getSaltingByte(dataRowKey, SaltingUtil.NUM_SALTING_BYTES, length - SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
dataRowKey[0] = saltByte;
}
return dataRowKey.length == length ? dataRowKey : Arrays.copyOf(dataRowKey, length);
} catch (IOException e) {
// Impossible
throw new RuntimeException(e);
} finally {
try {
stream.close();
} catch (IOException e) {
// Impossible
throw new RuntimeException(e);
}
}
}
Aggregations