Examples with RowMutationState org.apache.phoenix.execute.MutationState.RowMutationState used on opensource projects

Example 1 with RowMutationState

use of org.apache.phoenix.execute.MutationState.RowMutationState in project phoenix by apache.

the class UpsertCompiler method setValues.

private static void setValues(byte[][] values, int[] pkSlotIndex, int[] columnIndexes, PTable table, Map<ImmutableBytesPtr, RowMutationState> mutation, PhoenixStatement statement, boolean useServerTimestamp, IndexMaintainer maintainer, byte[][] viewConstants, byte[] onDupKeyBytes, int numSplColumns) throws SQLException {
    Map<PColumn, byte[]> columnValues = Maps.newHashMapWithExpectedSize(columnIndexes.length);
    byte[][] pkValues = new byte[table.getPKColumns().size()][];
    // here and we will fill in the byte later in PRowImpl.
    if (table.getBucketNum() != null) {
        pkValues[0] = new byte[] { 0 };
    }
    for (int i = 0; i < numSplColumns; i++) {
        pkValues[i + (table.getBucketNum() != null ? 1 : 0)] = values[i];
    }
    // case when the table doesn't have a row timestamp column
    Long rowTimestamp = null;
    RowTimestampColInfo rowTsColInfo = new RowTimestampColInfo(useServerTimestamp, rowTimestamp);
    for (int i = 0, j = numSplColumns; j < values.length; j++, i++) {
        byte[] value = values[j];
        PColumn column = table.getColumns().get(columnIndexes[i]);
        if (SchemaUtil.isPKColumn(column)) {
            pkValues[pkSlotIndex[i]] = value;
            if (SchemaUtil.getPKPosition(table, column) == table.getRowTimestampColPos()) {
                if (!useServerTimestamp) {
                    PColumn rowTimestampCol = table.getPKColumns().get(table.getRowTimestampColPos());
                    rowTimestamp = PLong.INSTANCE.getCodec().decodeLong(value, 0, rowTimestampCol.getSortOrder());
                    if (rowTimestamp < 0) {
                        throw new IllegalDataException("Value of a column designated as ROW_TIMESTAMP cannot be less than zero");
                    }
                    rowTsColInfo = new RowTimestampColInfo(useServerTimestamp, rowTimestamp);
                }
            }
        } else {
            columnValues.put(column, value);
        }
    }
    ImmutableBytesPtr ptr = new ImmutableBytesPtr();
    table.newKey(ptr, pkValues);
    if (table.getIndexType() == IndexType.LOCAL && maintainer != null) {
        byte[] rowKey = maintainer.buildDataRowKey(ptr, viewConstants);
        HRegionLocation region = statement.getConnection().getQueryServices().getTableRegionLocation(table.getParentName().getBytes(), rowKey);
        byte[] regionPrefix = region.getRegionInfo().getStartKey().length == 0 ? new byte[region.getRegionInfo().getEndKey().length] : region.getRegionInfo().getStartKey();
        if (regionPrefix.length != 0) {
            ptr.set(ScanRanges.prefixKey(ptr.get(), 0, regionPrefix, regionPrefix.length));
        }
    }
    mutation.put(ptr, new RowMutationState(columnValues, statement.getConnection().getStatementExecutionCounter(), rowTsColInfo, onDupKeyBytes));
}

Example 2 with RowMutationState

use of org.apache.phoenix.execute.MutationState.RowMutationState in project phoenix by apache.

the class DeleteCompiler method deleteRows.

private static MutationState deleteRows(StatementContext childContext, TableRef targetTableRef, List<TableRef> indexTableRefs, ResultIterator iterator, RowProjector projector, TableRef sourceTableRef) throws SQLException {
    PTable table = targetTableRef.getTable();
    PhoenixStatement statement = childContext.getStatement();
    PhoenixConnection connection = statement.getConnection();
    PName tenantId = connection.getTenantId();
    byte[] tenantIdBytes = null;
    if (tenantId != null) {
        tenantIdBytes = ScanUtil.getTenantIdBytes(table.getRowKeySchema(), table.getBucketNum() != null, tenantId, table.getViewIndexId() != null);
    }
    final boolean isAutoCommit = connection.getAutoCommit();
    ConnectionQueryServices services = connection.getQueryServices();
    final int maxSize = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
    final int maxSizeBytes = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE_BYTES);
    final int batchSize = Math.min(connection.getMutateBatchSize(), maxSize);
    Map<ImmutableBytesPtr, RowMutationState> mutations = Maps.newHashMapWithExpectedSize(batchSize);
    List<Map<ImmutableBytesPtr, RowMutationState>> indexMutations = null;
    // the data table through a single query to save executing an additional one.
    if (!indexTableRefs.isEmpty()) {
        indexMutations = Lists.newArrayListWithExpectedSize(indexTableRefs.size());
        for (int i = 0; i < indexTableRefs.size(); i++) {
            indexMutations.add(Maps.<ImmutableBytesPtr, RowMutationState>newHashMapWithExpectedSize(batchSize));
        }
    }
    List<PColumn> pkColumns = table.getPKColumns();
    boolean isMultiTenant = table.isMultiTenant() && tenantIdBytes != null;
    boolean isSharedViewIndex = table.getViewIndexId() != null;
    int offset = (table.getBucketNum() == null ? 0 : 1);
    byte[][] values = new byte[pkColumns.size()][];
    if (isSharedViewIndex) {
        values[offset++] = MetaDataUtil.getViewIndexIdDataType().toBytes(table.getViewIndexId());
    }
    if (isMultiTenant) {
        values[offset++] = tenantIdBytes;
    }
    try (PhoenixResultSet rs = new PhoenixResultSet(iterator, projector, childContext)) {
        int rowCount = 0;
        while (rs.next()) {
            // allocate new as this is a key in a Map
            ImmutableBytesPtr ptr = new ImmutableBytesPtr();
            // there's no transation required.
            if (sourceTableRef.equals(targetTableRef)) {
                rs.getCurrentRow().getKey(ptr);
            } else {
                for (int i = offset; i < values.length; i++) {
                    byte[] byteValue = rs.getBytes(i + 1 - offset);
                    // TODO: consider going under the hood and just getting the bytes
                    if (pkColumns.get(i).getSortOrder() == SortOrder.DESC) {
                        byte[] tempByteValue = Arrays.copyOf(byteValue, byteValue.length);
                        byteValue = SortOrder.invert(byteValue, 0, tempByteValue, 0, byteValue.length);
                    }
                    values[i] = byteValue;
                }
                table.newKey(ptr, values);
            }
            // When issuing deletes, we do not care about the row time ranges. Also, if the table had a row timestamp column, then the
            // row key will already have its value. 
            mutations.put(ptr, new RowMutationState(PRow.DELETE_MARKER, statement.getConnection().getStatementExecutionCounter(), NULL_ROWTIMESTAMP_INFO, null));
            for (int i = 0; i < indexTableRefs.size(); i++) {
                // allocate new as this is a key in a Map
                ImmutableBytesPtr indexPtr = new ImmutableBytesPtr();
                rs.getCurrentRow().getKey(indexPtr);
                indexMutations.get(i).put(indexPtr, new RowMutationState(PRow.DELETE_MARKER, statement.getConnection().getStatementExecutionCounter(), NULL_ROWTIMESTAMP_INFO, null));
            }
            if (mutations.size() > maxSize) {
                throw new IllegalArgumentException("MutationState size of " + mutations.size() + " is bigger than max allowed size of " + maxSize);
            }
            rowCount++;
            // Commit a batch if auto commit is true and we're at our batch size
            if (isAutoCommit && rowCount % batchSize == 0) {
                MutationState state = new MutationState(targetTableRef, mutations, 0, maxSize, maxSizeBytes, connection);
                connection.getMutationState().join(state);
                for (int i = 0; i < indexTableRefs.size(); i++) {
                    MutationState indexState = new MutationState(indexTableRefs.get(i), indexMutations.get(i), 0, maxSize, maxSizeBytes, connection);
                    connection.getMutationState().join(indexState);
                }
                connection.getMutationState().send();
                mutations.clear();
                if (indexMutations != null) {
                    indexMutations.clear();
                }
            }
        }
        // If auto commit is true, this last batch will be committed upon return
        int nCommittedRows = isAutoCommit ? (rowCount / batchSize * batchSize) : 0;
        MutationState state = new MutationState(targetTableRef, mutations, nCommittedRows, maxSize, maxSizeBytes, connection);
        for (int i = 0; i < indexTableRefs.size(); i++) {
            // To prevent the counting of these index rows, we have a negative for remainingRows.
            MutationState indexState = new MutationState(indexTableRefs.get(i), indexMutations.get(i), 0, maxSize, maxSizeBytes, connection);
            state.join(indexState);
        }
        return state;
    }
}

Example 3 with RowMutationState

use of org.apache.phoenix.execute.MutationState.RowMutationState in project phoenix by apache.

the class KeyValueUtil method getEstimatedRowSize.

/**
     * Estimates the storage size of a row
     * @param mutations map from table to row to RowMutationState
     * @return estimated row size
     */
public static long getEstimatedRowSize(Map<TableRef, Map<ImmutableBytesPtr, RowMutationState>> mutations) {
    long size = 0;
    // iterate over tables
    for (Entry<TableRef, Map<ImmutableBytesPtr, RowMutationState>> tableEntry : mutations.entrySet()) {
        PTable table = tableEntry.getKey().getTable();
        // iterate over rows
        for (Entry<ImmutableBytesPtr, RowMutationState> rowEntry : tableEntry.getValue().entrySet()) {
            int rowLength = rowEntry.getKey().getLength();
            Map<PColumn, byte[]> colValueMap = rowEntry.getValue().getColumnValues();
            switch(table.getImmutableStorageScheme()) {
                case ONE_CELL_PER_COLUMN:
                    // iterate over columns
                    for (Entry<PColumn, byte[]> colValueEntry : colValueMap.entrySet()) {
                        PColumn pColumn = colValueEntry.getKey();
                        size += KeyValue.getKeyValueDataStructureSize(rowLength, pColumn.getFamilyName().getBytes().length, pColumn.getColumnQualifierBytes().length, colValueEntry.getValue().length);
                    }
                    break;
                case SINGLE_CELL_ARRAY_WITH_OFFSETS:
                    // we store all the column values in a single key value that contains all the
                    // column values followed by an offset array
                    size += PArrayDataTypeEncoder.getEstimatedByteSize(table, rowLength, colValueMap);
                    break;
            }
            // count the empty key value
            Pair<byte[], byte[]> emptyKeyValueInfo = EncodedColumnsUtil.getEmptyKeyValueInfo(table);
            size += KeyValue.getKeyValueDataStructureSize(rowLength, SchemaUtil.getEmptyColumnFamilyPtr(table).getLength(), emptyKeyValueInfo.getFirst().length, emptyKeyValueInfo.getSecond().length);
        }
    }
    return size;
}

Example 4 with RowMutationState

use of org.apache.phoenix.execute.MutationState.RowMutationState in project phoenix by apache.

the class DeleteCompiler method deleteRows.

/**
 * Handles client side deletion of rows for a DELETE statement. We determine the "best" plan to drive the query using
 * our standard optimizer. The plan may be based on using an index, in which case we need to translate the index row
 * key to get the data row key used to form the delete mutation. We always collect up the data table mutations, but we
 * only collect and send the index mutations for global, immutable indexes. Local indexes and mutable indexes are always
 * maintained on the server side.
 * @param context StatementContext for the scan being executed
 * @param iterator ResultIterator for the scan being executed
 * @param bestPlan QueryPlan used to produce the iterator
 * @param projectedTableRef TableRef containing all indexed and covered columns across all indexes on the data table
 * @param otherTableRefs other TableRefs needed to be maintained apart from the one over which the scan is executing.
 *  Might be other index tables (if we're driving off of the data table table), the data table (if we're driving off of
 *  an index table), or a mix of the data table and additional index tables.
 * @return MutationState representing the uncommitted data across the data table and indexes. Will be joined with the
 *  MutationState on the connection over which the delete is occurring.
 * @throws SQLException
 */
private static MutationState deleteRows(StatementContext context, ResultIterator iterator, QueryPlan bestPlan, TableRef projectedTableRef, List<TableRef> otherTableRefs) throws SQLException {
    RowProjector projector = bestPlan.getProjector();
    TableRef tableRef = bestPlan.getTableRef();
    PTable table = tableRef.getTable();
    PhoenixStatement statement = context.getStatement();
    PhoenixConnection connection = statement.getConnection();
    PName tenantId = connection.getTenantId();
    byte[] tenantIdBytes = null;
    if (tenantId != null) {
        tenantIdBytes = ScanUtil.getTenantIdBytes(table.getRowKeySchema(), table.getBucketNum() != null, tenantId, table.getViewIndexId() != null);
    }
    final boolean isAutoCommit = connection.getAutoCommit();
    ConnectionQueryServices services = connection.getQueryServices();
    final int maxSize = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
    final int maxSizeBytes = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE_BYTES);
    final int batchSize = Math.min(connection.getMutateBatchSize(), maxSize);
    MultiRowMutationState mutations = new MultiRowMutationState(batchSize);
    List<MultiRowMutationState> otherMutations = null;
    // can always get the data table row key from an index row key).
    if (!otherTableRefs.isEmpty()) {
        otherMutations = Lists.newArrayListWithExpectedSize(otherTableRefs.size());
        for (int i = 0; i < otherTableRefs.size(); i++) {
            otherMutations.add(new MultiRowMutationState(batchSize));
        }
    }
    List<PColumn> pkColumns = table.getPKColumns();
    boolean isMultiTenant = table.isMultiTenant() && tenantIdBytes != null;
    boolean isSharedViewIndex = table.getViewIndexId() != null;
    int offset = (table.getBucketNum() == null ? 0 : 1);
    byte[][] values = new byte[pkColumns.size()][];
    if (isSharedViewIndex) {
        values[offset++] = MetaDataUtil.getViewIndexIdDataType().toBytes(table.getViewIndexId());
    }
    if (isMultiTenant) {
        values[offset++] = tenantIdBytes;
    }
    try (final PhoenixResultSet rs = new PhoenixResultSet(iterator, projector, context)) {
        ValueGetter getter = null;
        if (!otherTableRefs.isEmpty()) {
            getter = new ValueGetter() {

                final ImmutableBytesWritable valuePtr = new ImmutableBytesWritable();

                final ImmutableBytesWritable rowKeyPtr = new ImmutableBytesWritable();

                @Override
                public ImmutableBytesWritable getLatestValue(ColumnReference ref, long ts) throws IOException {
                    Cell cell = rs.getCurrentRow().getValue(ref.getFamily(), ref.getQualifier());
                    if (cell == null) {
                        return null;
                    }
                    valuePtr.set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
                    return valuePtr;
                }

                @Override
                public byte[] getRowKey() {
                    rs.getCurrentRow().getKey(rowKeyPtr);
                    return ByteUtil.copyKeyBytesIfNecessary(rowKeyPtr);
                }
            };
        }
        IndexMaintainer scannedIndexMaintainer = null;
        IndexMaintainer[] maintainers = null;
        PTable dataTable = table;
        if (table.getType() == PTableType.INDEX) {
            if (!otherTableRefs.isEmpty()) {
                // The data table is always the last one in the list if it's
                // not chosen as the best of the possible plans.
                dataTable = otherTableRefs.get(otherTableRefs.size() - 1).getTable();
                scannedIndexMaintainer = IndexMaintainer.create(dataTable, table, connection);
            }
            maintainers = new IndexMaintainer[otherTableRefs.size()];
            for (int i = 0; i < otherTableRefs.size(); i++) {
                // Create IndexMaintainer based on projected table (i.e. SELECT expressions) so that client-side
                // expressions are used instead of server-side ones.
                PTable otherTable = otherTableRefs.get(i).getTable();
                if (otherTable.getType() == PTableType.INDEX) {
                    // In this case, we'll convert from index row -> data row -> other index row
                    maintainers[i] = IndexMaintainer.create(dataTable, otherTable, connection);
                } else {
                    maintainers[i] = scannedIndexMaintainer;
                }
            }
        } else if (!otherTableRefs.isEmpty()) {
            dataTable = table;
            maintainers = new IndexMaintainer[otherTableRefs.size()];
            for (int i = 0; i < otherTableRefs.size(); i++) {
                // Create IndexMaintainer based on projected table (i.e. SELECT expressions) so that client-side
                // expressions are used instead of server-side ones.
                maintainers[i] = IndexMaintainer.create(projectedTableRef.getTable(), otherTableRefs.get(i).getTable(), connection);
            }
        }
        byte[][] viewConstants = IndexUtil.getViewConstants(dataTable);
        int rowCount = 0;
        while (rs.next()) {
            // allocate new as this is a key in a Map
            ImmutableBytesPtr rowKeyPtr = new ImmutableBytesPtr();
            rs.getCurrentRow().getKey(rowKeyPtr);
            // Check for otherTableRefs being empty required when deleting directly from the index
            if (otherTableRefs.isEmpty() || isMaintainedOnClient(table)) {
                mutations.put(rowKeyPtr, new RowMutationState(PRow.DELETE_MARKER, 0, statement.getConnection().getStatementExecutionCounter(), NULL_ROWTIMESTAMP_INFO, null));
            }
            for (int i = 0; i < otherTableRefs.size(); i++) {
                PTable otherTable = otherTableRefs.get(i).getTable();
                // allocate new as this is a key in a Map
                ImmutableBytesPtr otherRowKeyPtr = new ImmutableBytesPtr();
                // Translate the data table row to the index table row
                if (table.getType() == PTableType.INDEX) {
                    otherRowKeyPtr.set(scannedIndexMaintainer.buildDataRowKey(rowKeyPtr, viewConstants));
                    if (otherTable.getType() == PTableType.INDEX) {
                        otherRowKeyPtr.set(maintainers[i].buildRowKey(getter, otherRowKeyPtr, null, null, HConstants.LATEST_TIMESTAMP));
                    }
                } else {
                    otherRowKeyPtr.set(maintainers[i].buildRowKey(getter, rowKeyPtr, null, null, HConstants.LATEST_TIMESTAMP));
                }
                otherMutations.get(i).put(otherRowKeyPtr, new RowMutationState(PRow.DELETE_MARKER, 0, statement.getConnection().getStatementExecutionCounter(), NULL_ROWTIMESTAMP_INFO, null));
            }
            if (mutations.size() > maxSize) {
                throw new IllegalArgumentException("MutationState size of " + mutations.size() + " is bigger than max allowed size of " + maxSize);
            }
            rowCount++;
            // Commit a batch if auto commit is true and we're at our batch size
            if (isAutoCommit && rowCount % batchSize == 0) {
                MutationState state = new MutationState(tableRef, mutations, 0, maxSize, maxSizeBytes, connection);
                connection.getMutationState().join(state);
                for (int i = 0; i < otherTableRefs.size(); i++) {
                    MutationState indexState = new MutationState(otherTableRefs.get(i), otherMutations.get(i), 0, maxSize, maxSizeBytes, connection);
                    connection.getMutationState().join(indexState);
                }
                connection.getMutationState().send();
                mutations.clear();
                if (otherMutations != null) {
                    for (MultiRowMutationState multiRowMutationState : otherMutations) {
                        multiRowMutationState.clear();
                    }
                }
            }
        }
        // If auto commit is true, this last batch will be committed upon return
        int nCommittedRows = isAutoCommit ? (rowCount / batchSize * batchSize) : 0;
        MutationState state = new MutationState(tableRef, mutations, nCommittedRows, maxSize, maxSizeBytes, connection);
        for (int i = 0; i < otherTableRefs.size(); i++) {
            MutationState indexState = new MutationState(otherTableRefs.get(i), otherMutations.get(i), 0, maxSize, maxSizeBytes, connection);
            state.join(indexState);
        }
        return state;
    }
}

Example 5 with RowMutationState

use of org.apache.phoenix.execute.MutationState.RowMutationState in project phoenix by apache.

the class UpsertCompiler method setValues.

private static void setValues(byte[][] values, int[] pkSlotIndex, int[] columnIndexes, PTable table, MultiRowMutationState mutation, PhoenixStatement statement, boolean useServerTimestamp, IndexMaintainer maintainer, byte[][] viewConstants, byte[] onDupKeyBytes, int numSplColumns) throws SQLException {
    long columnValueSize = 0;
    Map<PColumn, byte[]> columnValues = Maps.newHashMapWithExpectedSize(columnIndexes.length);
    byte[][] pkValues = new byte[table.getPKColumns().size()][];
    // here and we will fill in the byte later in PRowImpl.
    if (table.getBucketNum() != null) {
        pkValues[0] = new byte[] { 0 };
    }
    for (int i = 0; i < numSplColumns; i++) {
        pkValues[i + (table.getBucketNum() != null ? 1 : 0)] = values[i];
    }
    // case when the table doesn't have a row timestamp column
    Long rowTimestamp = null;
    RowTimestampColInfo rowTsColInfo = new RowTimestampColInfo(useServerTimestamp, rowTimestamp);
    for (int i = 0, j = numSplColumns; j < values.length; j++, i++) {
        byte[] value = values[j];
        PColumn column = table.getColumns().get(columnIndexes[i]);
        if (SchemaUtil.isPKColumn(column)) {
            pkValues[pkSlotIndex[i]] = value;
            if (SchemaUtil.getPKPosition(table, column) == table.getRowTimestampColPos()) {
                if (!useServerTimestamp) {
                    PColumn rowTimestampCol = table.getPKColumns().get(table.getRowTimestampColPos());
                    rowTimestamp = PLong.INSTANCE.getCodec().decodeLong(value, 0, rowTimestampCol.getSortOrder());
                    if (rowTimestamp < 0) {
                        throw new IllegalDataException("Value of a column designated as ROW_TIMESTAMP cannot be less than zero");
                    }
                    rowTsColInfo = new RowTimestampColInfo(useServerTimestamp, rowTimestamp);
                }
            }
        } else {
            columnValues.put(column, value);
            columnValueSize += (column.getEstimatedSize() + value.length);
        }
    }
    ImmutableBytesPtr ptr = new ImmutableBytesPtr();
    table.newKey(ptr, pkValues);
    if (table.getIndexType() == IndexType.LOCAL && maintainer != null) {
        byte[] rowKey = maintainer.buildDataRowKey(ptr, viewConstants);
        HRegionLocation region = statement.getConnection().getQueryServices().getTableRegionLocation(table.getParentName().getBytes(), rowKey);
        byte[] regionPrefix = region.getRegionInfo().getStartKey().length == 0 ? new byte[region.getRegionInfo().getEndKey().length] : region.getRegionInfo().getStartKey();
        if (regionPrefix.length != 0) {
            ptr.set(ScanRanges.prefixKey(ptr.get(), 0, ptr.getLength(), regionPrefix, regionPrefix.length));
        }
    }
    mutation.put(ptr, new RowMutationState(columnValues, columnValueSize, statement.getConnection().getStatementExecutionCounter(), rowTsColInfo, onDupKeyBytes));
}