Examples with PName org.apache.phoenix.schema.PName used on opensource projects

Example 21 with PName

use of org.apache.phoenix.schema.PName in project phoenix by apache.

the class DeleteCompiler method deleteRows.

/**
 * Handles client side deletion of rows for a DELETE statement. We determine the "best" plan to drive the query using
 * our standard optimizer. The plan may be based on using an index, in which case we need to translate the index row
 * key to get the data row key used to form the delete mutation. We always collect up the data table mutations, but we
 * only collect and send the index mutations for global, immutable indexes. Local indexes and mutable indexes are always
 * maintained on the server side.
 * @param context StatementContext for the scan being executed
 * @param iterator ResultIterator for the scan being executed
 * @param bestPlan QueryPlan used to produce the iterator
 * @param projectedTableRef TableRef containing all indexed and covered columns across all indexes on the data table
 * @param otherTableRefs other TableRefs needed to be maintained apart from the one over which the scan is executing.
 *  Might be other index tables (if we're driving off of the data table table), the data table (if we're driving off of
 *  an index table), or a mix of the data table and additional index tables.
 * @return MutationState representing the uncommitted data across the data table and indexes. Will be joined with the
 *  MutationState on the connection over which the delete is occurring.
 * @throws SQLException
 */
private static MutationState deleteRows(StatementContext context, ResultIterator iterator, QueryPlan bestPlan, TableRef projectedTableRef, List<TableRef> otherTableRefs) throws SQLException {
    RowProjector projector = bestPlan.getProjector();
    TableRef tableRef = bestPlan.getTableRef();
    PTable table = tableRef.getTable();
    PhoenixStatement statement = context.getStatement();
    PhoenixConnection connection = statement.getConnection();
    PName tenantId = connection.getTenantId();
    byte[] tenantIdBytes = null;
    if (tenantId != null) {
        tenantIdBytes = ScanUtil.getTenantIdBytes(table.getRowKeySchema(), table.getBucketNum() != null, tenantId, table.getViewIndexId() != null);
    }
    final boolean isAutoCommit = connection.getAutoCommit();
    ConnectionQueryServices services = connection.getQueryServices();
    final int maxSize = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
    final int maxSizeBytes = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE_BYTES);
    final int batchSize = Math.min(connection.getMutateBatchSize(), maxSize);
    MultiRowMutationState mutations = new MultiRowMutationState(batchSize);
    List<MultiRowMutationState> otherMutations = null;
    // can always get the data table row key from an index row key).
    if (!otherTableRefs.isEmpty()) {
        otherMutations = Lists.newArrayListWithExpectedSize(otherTableRefs.size());
        for (int i = 0; i < otherTableRefs.size(); i++) {
            otherMutations.add(new MultiRowMutationState(batchSize));
        }
    }
    List<PColumn> pkColumns = table.getPKColumns();
    boolean isMultiTenant = table.isMultiTenant() && tenantIdBytes != null;
    boolean isSharedViewIndex = table.getViewIndexId() != null;
    int offset = (table.getBucketNum() == null ? 0 : 1);
    byte[][] values = new byte[pkColumns.size()][];
    if (isSharedViewIndex) {
        values[offset++] = MetaDataUtil.getViewIndexIdDataType().toBytes(table.getViewIndexId());
    }
    if (isMultiTenant) {
        values[offset++] = tenantIdBytes;
    }
    try (final PhoenixResultSet rs = new PhoenixResultSet(iterator, projector, context)) {
        ValueGetter getter = null;
        if (!otherTableRefs.isEmpty()) {
            getter = new ValueGetter() {

                final ImmutableBytesWritable valuePtr = new ImmutableBytesWritable();

                final ImmutableBytesWritable rowKeyPtr = new ImmutableBytesWritable();

                @Override
                public ImmutableBytesWritable getLatestValue(ColumnReference ref, long ts) throws IOException {
                    Cell cell = rs.getCurrentRow().getValue(ref.getFamily(), ref.getQualifier());
                    if (cell == null) {
                        return null;
                    }
                    valuePtr.set(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
                    return valuePtr;
                }

                @Override
                public byte[] getRowKey() {
                    rs.getCurrentRow().getKey(rowKeyPtr);
                    return ByteUtil.copyKeyBytesIfNecessary(rowKeyPtr);
                }
            };
        }
        IndexMaintainer scannedIndexMaintainer = null;
        IndexMaintainer[] maintainers = null;
        PTable dataTable = table;
        if (table.getType() == PTableType.INDEX) {
            if (!otherTableRefs.isEmpty()) {
                // The data table is always the last one in the list if it's
                // not chosen as the best of the possible plans.
                dataTable = otherTableRefs.get(otherTableRefs.size() - 1).getTable();
                scannedIndexMaintainer = IndexMaintainer.create(dataTable, table, connection);
            }
            maintainers = new IndexMaintainer[otherTableRefs.size()];
            for (int i = 0; i < otherTableRefs.size(); i++) {
                // Create IndexMaintainer based on projected table (i.e. SELECT expressions) so that client-side
                // expressions are used instead of server-side ones.
                PTable otherTable = otherTableRefs.get(i).getTable();
                if (otherTable.getType() == PTableType.INDEX) {
                    // In this case, we'll convert from index row -> data row -> other index row
                    maintainers[i] = IndexMaintainer.create(dataTable, otherTable, connection);
                } else {
                    maintainers[i] = scannedIndexMaintainer;
                }
            }
        } else if (!otherTableRefs.isEmpty()) {
            dataTable = table;
            maintainers = new IndexMaintainer[otherTableRefs.size()];
            for (int i = 0; i < otherTableRefs.size(); i++) {
                // Create IndexMaintainer based on projected table (i.e. SELECT expressions) so that client-side
                // expressions are used instead of server-side ones.
                maintainers[i] = IndexMaintainer.create(projectedTableRef.getTable(), otherTableRefs.get(i).getTable(), connection);
            }
        }
        byte[][] viewConstants = IndexUtil.getViewConstants(dataTable);
        int rowCount = 0;
        while (rs.next()) {
            // allocate new as this is a key in a Map
            ImmutableBytesPtr rowKeyPtr = new ImmutableBytesPtr();
            rs.getCurrentRow().getKey(rowKeyPtr);
            // Check for otherTableRefs being empty required when deleting directly from the index
            if (otherTableRefs.isEmpty() || isMaintainedOnClient(table)) {
                mutations.put(rowKeyPtr, new RowMutationState(PRow.DELETE_MARKER, 0, statement.getConnection().getStatementExecutionCounter(), NULL_ROWTIMESTAMP_INFO, null));
            }
            for (int i = 0; i < otherTableRefs.size(); i++) {
                PTable otherTable = otherTableRefs.get(i).getTable();
                // allocate new as this is a key in a Map
                ImmutableBytesPtr otherRowKeyPtr = new ImmutableBytesPtr();
                // Translate the data table row to the index table row
                if (table.getType() == PTableType.INDEX) {
                    otherRowKeyPtr.set(scannedIndexMaintainer.buildDataRowKey(rowKeyPtr, viewConstants));
                    if (otherTable.getType() == PTableType.INDEX) {
                        otherRowKeyPtr.set(maintainers[i].buildRowKey(getter, otherRowKeyPtr, null, null, HConstants.LATEST_TIMESTAMP));
                    }
                } else {
                    otherRowKeyPtr.set(maintainers[i].buildRowKey(getter, rowKeyPtr, null, null, HConstants.LATEST_TIMESTAMP));
                }
                otherMutations.get(i).put(otherRowKeyPtr, new RowMutationState(PRow.DELETE_MARKER, 0, statement.getConnection().getStatementExecutionCounter(), NULL_ROWTIMESTAMP_INFO, null));
            }
            if (mutations.size() > maxSize) {
                throw new IllegalArgumentException("MutationState size of " + mutations.size() + " is bigger than max allowed size of " + maxSize);
            }
            rowCount++;
            // Commit a batch if auto commit is true and we're at our batch size
            if (isAutoCommit && rowCount % batchSize == 0) {
                MutationState state = new MutationState(tableRef, mutations, 0, maxSize, maxSizeBytes, connection);
                connection.getMutationState().join(state);
                for (int i = 0; i < otherTableRefs.size(); i++) {
                    MutationState indexState = new MutationState(otherTableRefs.get(i), otherMutations.get(i), 0, maxSize, maxSizeBytes, connection);
                    connection.getMutationState().join(indexState);
                }
                connection.getMutationState().send();
                mutations.clear();
                if (otherMutations != null) {
                    for (MultiRowMutationState multiRowMutationState : otherMutations) {
                        multiRowMutationState.clear();
                    }
                }
            }
        }
        // If auto commit is true, this last batch will be committed upon return
        int nCommittedRows = isAutoCommit ? (rowCount / batchSize * batchSize) : 0;
        MutationState state = new MutationState(tableRef, mutations, nCommittedRows, maxSize, maxSizeBytes, connection);
        for (int i = 0; i < otherTableRefs.size(); i++) {
            MutationState indexState = new MutationState(otherTableRefs.get(i), otherMutations.get(i), 0, maxSize, maxSizeBytes, connection);
            state.join(indexState);
        }
        return state;
    }
}

Example 22 with PName

use of org.apache.phoenix.schema.PName in project phoenix by apache.

the class PhoenixConfigurationUtil method getStatsForParallelizationProp.

public static boolean getStatsForParallelizationProp(PhoenixConnection conn, PTable table) {
    Boolean useStats = table.useStatsForParallelization();
    if (useStats != null) {
        return useStats;
    }
    /*
	     * For a view index, we use the property set on view. For indexes on base table, whether
	     * global or local, we use the property set on the base table. Null check needed when
	     * dropping local indexes.
	     */
    PName tenantId = conn.getTenantId();
    int retryCount = 0;
    while (retryCount++ < 2) {
        if (table.getType() == PTableType.INDEX && table.getParentName() != null) {
            String parentTableName = table.getParentName().getString();
            try {
                PTable parentTable = conn.getTable(new PTableKey(tenantId, parentTableName));
                useStats = parentTable.useStatsForParallelization();
                if (useStats != null) {
                    return useStats;
                }
            } catch (TableNotFoundException e) {
                // global tables)
                if (tenantId != null) {
                    tenantId = null;
                } else {
                    BaseResultIterators.logger.warn("Unable to find parent table \"" + parentTableName + "\" of table \"" + table.getName().getString() + "\" to determine USE_STATS_FOR_PARALLELIZATION", e);
                }
            }
        }
    }
    return conn.getQueryServices().getConfiguration().getBoolean(USE_STATS_FOR_PARALLELIZATION, DEFAULT_USE_STATS_FOR_PARALLELIZATION);
}

Example 23 with PName

use of org.apache.phoenix.schema.PName in project phoenix by apache.

the class MetaDataEndpointImpl method getTable.

private PTable getTable(RegionScanner scanner, long clientTimeStamp, long tableTimeStamp, int clientVersion) throws IOException, SQLException {
    List<Cell> results = Lists.newArrayList();
    scanner.next(results);
    if (results.isEmpty()) {
        return null;
    }
    Cell[] tableKeyValues = new Cell[TABLE_KV_COLUMNS.size()];
    Cell[] colKeyValues = new Cell[COLUMN_KV_COLUMNS.size()];
    // Create PTable based on KeyValues from scan
    Cell keyValue = results.get(0);
    byte[] keyBuffer = keyValue.getRowArray();
    int keyLength = keyValue.getRowLength();
    int keyOffset = keyValue.getRowOffset();
    PName tenantId = newPName(keyBuffer, keyOffset, keyLength);
    int tenantIdLength = (tenantId == null) ? 0 : tenantId.getBytes().length;
    if (tenantIdLength == 0) {
        tenantId = null;
    }
    PName schemaName = newPName(keyBuffer, keyOffset + tenantIdLength + 1, keyLength);
    int schemaNameLength = schemaName.getBytes().length;
    int tableNameLength = keyLength - schemaNameLength - 1 - tenantIdLength - 1;
    byte[] tableNameBytes = new byte[tableNameLength];
    System.arraycopy(keyBuffer, keyOffset + schemaNameLength + 1 + tenantIdLength + 1, tableNameBytes, 0, tableNameLength);
    PName tableName = PNameFactory.newName(tableNameBytes);
    int offset = tenantIdLength + schemaNameLength + tableNameLength + 3;
    // This will prevent the client from continually looking for the current
    // table when we know that there will never be one since we disallow updates
    // unless the table is the latest
    // If we already have a table newer than the one we just found and
    // the client timestamp is less that the existing table time stamp,
    // bump up the timeStamp to right before the client time stamp, since
    // we know it can't possibly change.
    long timeStamp = keyValue.getTimestamp();
    // long timeStamp = tableTimeStamp > keyValue.getTimestamp() &&
    // clientTimeStamp < tableTimeStamp
    // ? clientTimeStamp-1
    // : keyValue.getTimestamp();
    int i = 0;
    int j = 0;
    while (i < results.size() && j < TABLE_KV_COLUMNS.size()) {
        Cell kv = results.get(i);
        Cell searchKv = TABLE_KV_COLUMNS.get(j);
        int cmp = Bytes.compareTo(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(), searchKv.getQualifierArray(), searchKv.getQualifierOffset(), searchKv.getQualifierLength());
        if (cmp == 0) {
            // Find max timestamp of table
            timeStamp = Math.max(timeStamp, kv.getTimestamp());
            // header row
            tableKeyValues[j++] = kv;
            i++;
        } else if (cmp > 0) {
            timeStamp = Math.max(timeStamp, kv.getTimestamp());
            tableKeyValues[j++] = null;
        } else {
            // shouldn't happen - means unexpected KV in system table header row
            i++;
        }
    }
    // TABLE_TYPE, TABLE_SEQ_NUM and COLUMN_COUNT are required.
    if (tableKeyValues[TABLE_TYPE_INDEX] == null || tableKeyValues[TABLE_SEQ_NUM_INDEX] == null || tableKeyValues[COLUMN_COUNT_INDEX] == null) {
        throw new IllegalStateException("Didn't find expected key values for table row in metadata row");
    }
    Cell tableTypeKv = tableKeyValues[TABLE_TYPE_INDEX];
    PTableType tableType = PTableType.fromSerializedValue(tableTypeKv.getValueArray()[tableTypeKv.getValueOffset()]);
    Cell tableSeqNumKv = tableKeyValues[TABLE_SEQ_NUM_INDEX];
    long tableSeqNum = PLong.INSTANCE.getCodec().decodeLong(tableSeqNumKv.getValueArray(), tableSeqNumKv.getValueOffset(), SortOrder.getDefault());
    Cell columnCountKv = tableKeyValues[COLUMN_COUNT_INDEX];
    int columnCount = PInteger.INSTANCE.getCodec().decodeInt(columnCountKv.getValueArray(), columnCountKv.getValueOffset(), SortOrder.getDefault());
    Cell pkNameKv = tableKeyValues[PK_NAME_INDEX];
    PName pkName = pkNameKv != null ? newPName(pkNameKv.getValueArray(), pkNameKv.getValueOffset(), pkNameKv.getValueLength()) : null;
    Cell saltBucketNumKv = tableKeyValues[SALT_BUCKETS_INDEX];
    Integer saltBucketNum = saltBucketNumKv != null ? (Integer) PInteger.INSTANCE.getCodec().decodeInt(saltBucketNumKv.getValueArray(), saltBucketNumKv.getValueOffset(), SortOrder.getDefault()) : null;
    if (saltBucketNum != null && saltBucketNum.intValue() == 0) {
        // Zero salt buckets means not salted
        saltBucketNum = null;
    }
    Cell dataTableNameKv = tableKeyValues[DATA_TABLE_NAME_INDEX];
    PName dataTableName = dataTableNameKv != null ? newPName(dataTableNameKv.getValueArray(), dataTableNameKv.getValueOffset(), dataTableNameKv.getValueLength()) : null;
    Cell indexStateKv = tableKeyValues[INDEX_STATE_INDEX];
    PIndexState indexState = indexStateKv == null ? null : PIndexState.fromSerializedValue(indexStateKv.getValueArray()[indexStateKv.getValueOffset()]);
    // the value in those versions) since the client won't have this index state in its enum.
    if (indexState == PIndexState.PENDING_ACTIVE && clientVersion < PhoenixDatabaseMetaData.MIN_PENDING_ACTIVE_INDEX) {
        indexState = PIndexState.ACTIVE;
    }
    // since the client won't have this index state in its enum.
    if (indexState == PIndexState.PENDING_DISABLE && clientVersion < PhoenixDatabaseMetaData.MIN_PENDING_DISABLE_INDEX) {
        // note: for older clients, we have to rely on the rebuilder to transition PENDING_DISABLE -> DISABLE
        indexState = PIndexState.DISABLE;
    }
    Cell immutableRowsKv = tableKeyValues[IMMUTABLE_ROWS_INDEX];
    boolean isImmutableRows = immutableRowsKv == null ? false : (Boolean) PBoolean.INSTANCE.toObject(immutableRowsKv.getValueArray(), immutableRowsKv.getValueOffset(), immutableRowsKv.getValueLength());
    Cell defaultFamilyNameKv = tableKeyValues[DEFAULT_COLUMN_FAMILY_INDEX];
    PName defaultFamilyName = defaultFamilyNameKv != null ? newPName(defaultFamilyNameKv.getValueArray(), defaultFamilyNameKv.getValueOffset(), defaultFamilyNameKv.getValueLength()) : null;
    Cell viewStatementKv = tableKeyValues[VIEW_STATEMENT_INDEX];
    String viewStatement = viewStatementKv != null ? (String) PVarchar.INSTANCE.toObject(viewStatementKv.getValueArray(), viewStatementKv.getValueOffset(), viewStatementKv.getValueLength()) : null;
    Cell disableWALKv = tableKeyValues[DISABLE_WAL_INDEX];
    boolean disableWAL = disableWALKv == null ? PTable.DEFAULT_DISABLE_WAL : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(disableWALKv.getValueArray(), disableWALKv.getValueOffset(), disableWALKv.getValueLength()));
    Cell multiTenantKv = tableKeyValues[MULTI_TENANT_INDEX];
    boolean multiTenant = multiTenantKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(multiTenantKv.getValueArray(), multiTenantKv.getValueOffset(), multiTenantKv.getValueLength()));
    Cell storeNullsKv = tableKeyValues[STORE_NULLS_INDEX];
    boolean storeNulls = storeNullsKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(storeNullsKv.getValueArray(), storeNullsKv.getValueOffset(), storeNullsKv.getValueLength()));
    Cell transactionalKv = tableKeyValues[TRANSACTIONAL_INDEX];
    boolean transactional = transactionalKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(transactionalKv.getValueArray(), transactionalKv.getValueOffset(), transactionalKv.getValueLength()));
    Cell viewTypeKv = tableKeyValues[VIEW_TYPE_INDEX];
    ViewType viewType = viewTypeKv == null ? null : ViewType.fromSerializedValue(viewTypeKv.getValueArray()[viewTypeKv.getValueOffset()]);
    Cell viewIndexIdKv = tableKeyValues[VIEW_INDEX_ID_INDEX];
    Short viewIndexId = viewIndexIdKv == null ? null : (Short) MetaDataUtil.getViewIndexIdDataType().getCodec().decodeShort(viewIndexIdKv.getValueArray(), viewIndexIdKv.getValueOffset(), SortOrder.getDefault());
    Cell indexTypeKv = tableKeyValues[INDEX_TYPE_INDEX];
    IndexType indexType = indexTypeKv == null ? null : IndexType.fromSerializedValue(indexTypeKv.getValueArray()[indexTypeKv.getValueOffset()]);
    Cell baseColumnCountKv = tableKeyValues[BASE_COLUMN_COUNT_INDEX];
    int baseColumnCount = baseColumnCountKv == null ? 0 : PInteger.INSTANCE.getCodec().decodeInt(baseColumnCountKv.getValueArray(), baseColumnCountKv.getValueOffset(), SortOrder.getDefault());
    Cell rowKeyOrderOptimizableKv = tableKeyValues[ROW_KEY_ORDER_OPTIMIZABLE_INDEX];
    boolean rowKeyOrderOptimizable = rowKeyOrderOptimizableKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(rowKeyOrderOptimizableKv.getValueArray(), rowKeyOrderOptimizableKv.getValueOffset(), rowKeyOrderOptimizableKv.getValueLength()));
    Cell updateCacheFrequencyKv = tableKeyValues[UPDATE_CACHE_FREQUENCY_INDEX];
    long updateCacheFrequency = updateCacheFrequencyKv == null ? 0 : PLong.INSTANCE.getCodec().decodeLong(updateCacheFrequencyKv.getValueArray(), updateCacheFrequencyKv.getValueOffset(), SortOrder.getDefault());
    Cell indexDisableTimestampKv = tableKeyValues[INDEX_DISABLE_TIMESTAMP];
    long indexDisableTimestamp = indexDisableTimestampKv == null ? 0L : PLong.INSTANCE.getCodec().decodeLong(indexDisableTimestampKv.getValueArray(), indexDisableTimestampKv.getValueOffset(), SortOrder.getDefault());
    Cell isNamespaceMappedKv = tableKeyValues[IS_NAMESPACE_MAPPED_INDEX];
    boolean isNamespaceMapped = isNamespaceMappedKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(isNamespaceMappedKv.getValueArray(), isNamespaceMappedKv.getValueOffset(), isNamespaceMappedKv.getValueLength()));
    Cell autoPartitionSeqKv = tableKeyValues[AUTO_PARTITION_SEQ_INDEX];
    String autoPartitionSeq = autoPartitionSeqKv != null ? (String) PVarchar.INSTANCE.toObject(autoPartitionSeqKv.getValueArray(), autoPartitionSeqKv.getValueOffset(), autoPartitionSeqKv.getValueLength()) : null;
    Cell isAppendOnlySchemaKv = tableKeyValues[APPEND_ONLY_SCHEMA_INDEX];
    boolean isAppendOnlySchema = isAppendOnlySchemaKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(isAppendOnlySchemaKv.getValueArray(), isAppendOnlySchemaKv.getValueOffset(), isAppendOnlySchemaKv.getValueLength()));
    Cell storageSchemeKv = tableKeyValues[STORAGE_SCHEME_INDEX];
    // TODO: change this once we start having other values for storage schemes
    ImmutableStorageScheme storageScheme = storageSchemeKv == null ? ImmutableStorageScheme.ONE_CELL_PER_COLUMN : ImmutableStorageScheme.fromSerializedValue((byte) PTinyint.INSTANCE.toObject(storageSchemeKv.getValueArray(), storageSchemeKv.getValueOffset(), storageSchemeKv.getValueLength()));
    Cell encodingSchemeKv = tableKeyValues[QUALIFIER_ENCODING_SCHEME_INDEX];
    QualifierEncodingScheme encodingScheme = encodingSchemeKv == null ? QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : QualifierEncodingScheme.fromSerializedValue((byte) PTinyint.INSTANCE.toObject(encodingSchemeKv.getValueArray(), encodingSchemeKv.getValueOffset(), encodingSchemeKv.getValueLength()));
    Cell useStatsForParallelizationKv = tableKeyValues[USE_STATS_FOR_PARALLELIZATION_INDEX];
    Boolean useStatsForParallelization = useStatsForParallelizationKv == null ? null : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(useStatsForParallelizationKv.getValueArray(), useStatsForParallelizationKv.getValueOffset(), useStatsForParallelizationKv.getValueLength()));
    List<PColumn> columns = Lists.newArrayListWithExpectedSize(columnCount);
    List<PTable> indexes = Lists.newArrayList();
    List<PName> physicalTables = Lists.newArrayList();
    PName parentTableName = tableType == INDEX ? dataTableName : null;
    PName parentSchemaName = tableType == INDEX ? schemaName : null;
    EncodedCQCounter cqCounter = (!EncodedColumnsUtil.usesEncodedColumnNames(encodingScheme) || tableType == PTableType.VIEW) ? PTable.EncodedCQCounter.NULL_COUNTER : new EncodedCQCounter();
    while (true) {
        results.clear();
        scanner.next(results);
        if (results.isEmpty()) {
            break;
        }
        Cell colKv = results.get(LINK_TYPE_INDEX);
        int colKeyLength = colKv.getRowLength();
        PName colName = newPName(colKv.getRowArray(), colKv.getRowOffset() + offset, colKeyLength - offset);
        int colKeyOffset = offset + colName.getBytes().length + 1;
        PName famName = newPName(colKv.getRowArray(), colKv.getRowOffset() + colKeyOffset, colKeyLength - colKeyOffset);
        if (isQualifierCounterKV(colKv)) {
            Integer value = PInteger.INSTANCE.getCodec().decodeInt(colKv.getValueArray(), colKv.getValueOffset(), SortOrder.ASC);
            cqCounter.setValue(famName.getString(), value);
        } else if (Bytes.compareTo(LINK_TYPE_BYTES, 0, LINK_TYPE_BYTES.length, colKv.getQualifierArray(), colKv.getQualifierOffset(), colKv.getQualifierLength()) == 0) {
            LinkType linkType = LinkType.fromSerializedValue(colKv.getValueArray()[colKv.getValueOffset()]);
            if (linkType == LinkType.INDEX_TABLE) {
                addIndexToTable(tenantId, schemaName, famName, tableName, clientTimeStamp, indexes, clientVersion);
            } else if (linkType == LinkType.PHYSICAL_TABLE) {
                physicalTables.add(famName);
            } else if (linkType == LinkType.PARENT_TABLE) {
                parentTableName = PNameFactory.newName(SchemaUtil.getTableNameFromFullName(famName.getBytes()));
                parentSchemaName = PNameFactory.newName(SchemaUtil.getSchemaNameFromFullName(famName.getBytes()));
            }
        } else {
            addColumnToTable(results, colName, famName, colKeyValues, columns, saltBucketNum != null);
        }
    }
    // server while holding this lock is a bad idea and likely to cause contention.
    return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState, timeStamp, tableSeqNum, pkName, saltBucketNum, columns, parentSchemaName, parentTableName, indexes, isImmutableRows, physicalTables, defaultFamilyName, viewStatement, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, rowKeyOrderOptimizable, transactional, updateCacheFrequency, baseColumnCount, indexDisableTimestamp, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, storageScheme, encodingScheme, cqCounter, useStatsForParallelization);
}

Example 24 with PName

use of org.apache.phoenix.schema.PName in project phoenix by apache.

the class TupleProjectionCompiler method createProjectedTable.

public static PTable createProjectedTable(TableRef tableRef, List<ColumnRef> sourceColumnRefs, boolean retainPKColumns) throws SQLException {
    PTable table = tableRef.getTable();
    List<PColumn> projectedColumns = new ArrayList<PColumn>();
    int position = table.getBucketNum() != null ? 1 : 0;
    for (int i = retainPKColumns ? position : 0; i < sourceColumnRefs.size(); i++) {
        ColumnRef sourceColumnRef = sourceColumnRefs.get(i);
        PColumn sourceColumn = sourceColumnRef.getColumn();
        String colName = sourceColumn.getName().getString();
        String aliasedName = tableRef.getTableAlias() == null ? SchemaUtil.getColumnName(table.getName().getString(), colName) : SchemaUtil.getColumnName(tableRef.getTableAlias(), colName);
        PName familyName = SchemaUtil.isPKColumn(sourceColumn) ? (retainPKColumns ? null : PNameFactory.newName(VALUE_COLUMN_FAMILY)) : sourceColumn.getFamilyName();
        // If we're not retaining the PK columns, then we should switch columns to be nullable
        PColumn column = new ProjectedColumn(PNameFactory.newName(aliasedName), familyName, position++, sourceColumn.isNullable(), sourceColumnRef, sourceColumn.getColumnQualifierBytes());
        projectedColumns.add(column);
    }
    EncodedCQCounter cqCounter = EncodedCQCounter.NULL_COUNTER;
    if (EncodedColumnsUtil.usesEncodedColumnNames(table)) {
        cqCounter = EncodedCQCounter.copy(table.getEncodedCQCounter());
    }
    return PTableImpl.makePTable(table.getTenantId(), PROJECTED_TABLE_SCHEMA, table.getName(), PTableType.PROJECTED, null, table.getTimeStamp(), table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), projectedColumns, null, null, Collections.<PTable>emptyList(), table.isImmutableRows(), Collections.<PName>emptyList(), null, null, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), null, table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), cqCounter, table.useStatsForParallelization());
}

Example 25 with PName

use of org.apache.phoenix.schema.PName in project phoenix by apache.

the class UpsertCompiler method compile.

public MutationPlan compile(UpsertStatement upsert) throws SQLException {
    final PhoenixConnection connection = statement.getConnection();
    ConnectionQueryServices services = connection.getQueryServices();
    final int maxSize = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
    final int maxSizeBytes = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_BYTES_ATTRIB, QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE_BYTES);
    List<ColumnName> columnNodes = upsert.getColumns();
    TableRef tableRefToBe = null;
    PTable table = null;
    Set<PColumn> addViewColumnsToBe = Collections.emptySet();
    Set<PColumn> overlapViewColumnsToBe = Collections.emptySet();
    List<PColumn> allColumnsToBe = Collections.emptyList();
    boolean isTenantSpecific = false;
    boolean isSharedViewIndex = false;
    String tenantIdStr = null;
    ColumnResolver resolver = null;
    int[] columnIndexesToBe;
    int nColumnsToSet = 0;
    int[] pkSlotIndexesToBe;
    List<ParseNode> valueNodes = upsert.getValues();
    List<PColumn> targetColumns;
    NamedTableNode tableNode = upsert.getTable();
    String tableName = tableNode.getName().getTableName();
    String schemaName = tableNode.getName().getSchemaName();
    QueryPlan queryPlanToBe = null;
    int nValuesToSet;
    boolean sameTable = false;
    boolean runOnServer = false;
    boolean serverUpsertSelectEnabled = services.getProps().getBoolean(QueryServices.ENABLE_SERVER_UPSERT_SELECT, QueryServicesOptions.DEFAULT_ENABLE_SERVER_UPSERT_SELECT);
    UpsertingParallelIteratorFactory parallelIteratorFactoryToBe = null;
    boolean useServerTimestampToBe = false;
    resolver = FromCompiler.getResolverForMutation(upsert, connection);
    tableRefToBe = resolver.getTables().get(0);
    table = tableRefToBe.getTable();
    // - transactional table with a connection having an SCN
    if (table.getType() == PTableType.VIEW && table.getViewType().isReadOnly()) {
        throw new ReadOnlyTableException(schemaName, tableName);
    } else if (connection.isBuildingIndex() && table.getType() != PTableType.INDEX) {
        throw new SQLExceptionInfo.Builder(SQLExceptionCode.ONLY_INDEX_UPDATABLE_AT_SCN).setSchemaName(schemaName).setTableName(tableName).build().buildException();
    } else if (table.isTransactional() && connection.getSCN() != null) {
        throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_SPECIFY_SCN_FOR_TXN_TABLE).setSchemaName(schemaName).setTableName(tableName).build().buildException();
    }
    boolean isSalted = table.getBucketNum() != null;
    isTenantSpecific = table.isMultiTenant() && connection.getTenantId() != null;
    isSharedViewIndex = table.getViewIndexId() != null;
    tenantIdStr = isTenantSpecific ? connection.getTenantId().getString() : null;
    int posOffset = isSalted ? 1 : 0;
    // Setup array of column indexes parallel to values that are going to be set
    allColumnsToBe = table.getColumns();
    nColumnsToSet = 0;
    if (table.getViewType() == ViewType.UPDATABLE) {
        addViewColumnsToBe = Sets.newLinkedHashSetWithExpectedSize(allColumnsToBe.size());
        for (PColumn column : allColumnsToBe) {
            if (column.getViewConstant() != null) {
                addViewColumnsToBe.add(column);
            }
        }
    }
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    // Allow full row upsert if no columns or only dynamic ones are specified and values count match
    if (columnNodes.isEmpty() || columnNodes.size() == upsert.getTable().getDynamicColumns().size()) {
        nColumnsToSet = allColumnsToBe.size() - posOffset;
        columnIndexesToBe = new int[nColumnsToSet];
        pkSlotIndexesToBe = new int[columnIndexesToBe.length];
        targetColumns = Lists.newArrayListWithExpectedSize(columnIndexesToBe.length);
        targetColumns.addAll(Collections.<PColumn>nCopies(columnIndexesToBe.length, null));
        int minPKPos = 0;
        if (isSharedViewIndex) {
            PColumn indexIdColumn = table.getPKColumns().get(minPKPos);
            columnIndexesToBe[minPKPos] = indexIdColumn.getPosition();
            targetColumns.set(minPKPos, indexIdColumn);
            minPKPos++;
        }
        if (isTenantSpecific) {
            PColumn tenantColumn = table.getPKColumns().get(minPKPos);
            columnIndexesToBe[minPKPos] = tenantColumn.getPosition();
            targetColumns.set(minPKPos, tenantColumn);
            minPKPos++;
        }
        for (int i = posOffset, j = 0; i < allColumnsToBe.size(); i++) {
            PColumn column = allColumnsToBe.get(i);
            if (SchemaUtil.isPKColumn(column)) {
                pkSlotIndexesToBe[i - posOffset] = j + posOffset;
                if (j++ < minPKPos) {
                    // Skip, as it's already been set above
                    continue;
                }
                minPKPos = 0;
            }
            columnIndexesToBe[i - posOffset + minPKPos] = i;
            targetColumns.set(i - posOffset + minPKPos, column);
        }
        if (!addViewColumnsToBe.isEmpty()) {
            // All view columns overlap in this case
            overlapViewColumnsToBe = addViewColumnsToBe;
            addViewColumnsToBe = Collections.emptySet();
        }
    } else {
        // Size for worse case
        int numColsInUpsert = columnNodes.size();
        nColumnsToSet = numColsInUpsert + addViewColumnsToBe.size() + (isTenantSpecific ? 1 : 0) + +(isSharedViewIndex ? 1 : 0);
        columnIndexesToBe = new int[nColumnsToSet];
        pkSlotIndexesToBe = new int[columnIndexesToBe.length];
        targetColumns = Lists.newArrayListWithExpectedSize(columnIndexesToBe.length);
        targetColumns.addAll(Collections.<PColumn>nCopies(columnIndexesToBe.length, null));
        // TODO: necessary? So we'll get an AIOB exception if it's not replaced
        Arrays.fill(columnIndexesToBe, -1);
        // TODO: necessary? So we'll get an AIOB exception if it's not replaced
        Arrays.fill(pkSlotIndexesToBe, -1);
        BitSet columnsBeingSet = new BitSet(table.getColumns().size());
        int i = 0;
        if (isSharedViewIndex) {
            PColumn indexIdColumn = table.getPKColumns().get(i + posOffset);
            columnsBeingSet.set(columnIndexesToBe[i] = indexIdColumn.getPosition());
            pkSlotIndexesToBe[i] = i + posOffset;
            targetColumns.set(i, indexIdColumn);
            i++;
        }
        // Add tenant column directly, as we don't want to resolve it as this will fail
        if (isTenantSpecific) {
            PColumn tenantColumn = table.getPKColumns().get(i + posOffset);
            columnsBeingSet.set(columnIndexesToBe[i] = tenantColumn.getPosition());
            pkSlotIndexesToBe[i] = i + posOffset;
            targetColumns.set(i, tenantColumn);
            i++;
        }
        for (ColumnName colName : columnNodes) {
            ColumnRef ref = resolver.resolveColumn(null, colName.getFamilyName(), colName.getColumnName());
            PColumn column = ref.getColumn();
            if (IndexUtil.getViewConstantValue(column, ptr)) {
                if (overlapViewColumnsToBe.isEmpty()) {
                    overlapViewColumnsToBe = Sets.newHashSetWithExpectedSize(addViewColumnsToBe.size());
                }
                nColumnsToSet--;
                overlapViewColumnsToBe.add(column);
                addViewColumnsToBe.remove(column);
            }
            columnsBeingSet.set(columnIndexesToBe[i] = ref.getColumnPosition());
            targetColumns.set(i, column);
            if (SchemaUtil.isPKColumn(column)) {
                pkSlotIndexesToBe[i] = ref.getPKSlotPosition();
            }
            i++;
        }
        for (PColumn column : addViewColumnsToBe) {
            columnsBeingSet.set(columnIndexesToBe[i] = column.getPosition());
            targetColumns.set(i, column);
            if (SchemaUtil.isPKColumn(column)) {
                pkSlotIndexesToBe[i] = SchemaUtil.getPKPosition(table, column);
            }
            i++;
        }
        // If a table has rowtimestamp col, then we always set it.
        useServerTimestampToBe = table.getRowTimestampColPos() != -1 && !isRowTimestampSet(pkSlotIndexesToBe, table);
        if (useServerTimestampToBe) {
            PColumn rowTimestampCol = table.getPKColumns().get(table.getRowTimestampColPos());
            // Need to resize columnIndexesToBe and pkSlotIndexesToBe to include this extra column.
            columnIndexesToBe = Arrays.copyOf(columnIndexesToBe, columnIndexesToBe.length + 1);
            pkSlotIndexesToBe = Arrays.copyOf(pkSlotIndexesToBe, pkSlotIndexesToBe.length + 1);
            columnsBeingSet.set(columnIndexesToBe[i] = rowTimestampCol.getPosition());
            pkSlotIndexesToBe[i] = table.getRowTimestampColPos();
            targetColumns.add(rowTimestampCol);
            if (valueNodes != null && !valueNodes.isEmpty()) {
                valueNodes.add(getNodeForRowTimestampColumn(rowTimestampCol));
            }
            nColumnsToSet++;
        }
        for (i = posOffset; i < table.getColumns().size(); i++) {
            PColumn column = table.getColumns().get(i);
            if (!columnsBeingSet.get(i) && !column.isNullable() && column.getExpressionStr() == null) {
                throw new ConstraintViolationException(SchemaUtil.getColumnDisplayName(column) + " may not be null");
            }
        }
    }
    boolean isAutoCommit = connection.getAutoCommit();
    if (valueNodes == null) {
        SelectStatement select = upsert.getSelect();
        assert (select != null);
        select = SubselectRewriter.flatten(select, connection);
        ColumnResolver selectResolver = FromCompiler.getResolverForQuery(select, connection, false, upsert.getTable().getName());
        select = StatementNormalizer.normalize(select, selectResolver);
        select = prependTenantAndViewConstants(table, select, tenantIdStr, addViewColumnsToBe, useServerTimestampToBe);
        SelectStatement transformedSelect = SubqueryRewriter.transform(select, selectResolver, connection);
        if (transformedSelect != select) {
            selectResolver = FromCompiler.getResolverForQuery(transformedSelect, connection, false, upsert.getTable().getName());
            select = StatementNormalizer.normalize(transformedSelect, selectResolver);
        }
        sameTable = !select.isJoin() && tableRefToBe.equals(selectResolver.getTables().get(0));
        /* We can run the upsert in a coprocessor if:
             * 1) from has only 1 table or server UPSERT SELECT is enabled
             * 2) the select query isn't doing aggregation (which requires a client-side final merge)
             * 3) autoCommit is on
             * 4) the table is not immutable with indexes, as the client is the one that figures out the additional
             *    puts for index tables.
             * 5) no limit clause, as the limit clause requires client-side post processing
             * 6) no sequences, as sequences imply that the order of upsert must match the order of
             *    selection. TODO: change this and only force client side if there's a ORDER BY on the sequence value
             * Otherwise, run the query to pull the data from the server
             * and populate the MutationState (upto a limit).
            */
        if (!(select.isAggregate() || select.isDistinct() || select.getLimit() != null || select.hasSequence())) {
            // We can pipeline the upsert select instead of spooling everything to disk first,
            // if we don't have any post processing that's required.
            parallelIteratorFactoryToBe = new UpsertingParallelIteratorFactory(connection, tableRefToBe, useServerTimestampToBe);
            // If we're in the else, then it's not an aggregate, distinct, limited, or sequence using query,
            // so we might be able to run it entirely on the server side.
            // region space managed by region servers. So we bail out on executing on server side.
            // Disable running upsert select on server side if a table has global mutable secondary indexes on it
            boolean hasGlobalMutableIndexes = SchemaUtil.hasGlobalIndex(table) && !table.isImmutableRows();
            boolean hasWhereSubquery = select.getWhere() != null && select.getWhere().hasSubquery();
            runOnServer = (sameTable || (serverUpsertSelectEnabled && !hasGlobalMutableIndexes)) && isAutoCommit && !table.isTransactional() && !(table.isImmutableRows() && !table.getIndexes().isEmpty()) && !select.isJoin() && !hasWhereSubquery && table.getRowTimestampColPos() == -1;
        }
        // If we may be able to run on the server, add a hint that favors using the data table
        // if all else is equal.
        // TODO: it'd be nice if we could figure out in advance if the PK is potentially changing,
        // as this would disallow running on the server. We currently use the row projector we
        // get back to figure this out.
        HintNode hint = upsert.getHint();
        if (!upsert.getHint().hasHint(Hint.USE_INDEX_OVER_DATA_TABLE)) {
            hint = HintNode.create(hint, Hint.USE_DATA_OVER_INDEX_TABLE);
        }
        select = SelectStatement.create(select, hint);
        // Pass scan through if same table in upsert and select so that projection is computed correctly
        // Use optimizer to choose the best plan
        QueryCompiler compiler = new QueryCompiler(statement, select, selectResolver, targetColumns, parallelIteratorFactoryToBe, new SequenceManager(statement), false, false, null);
        queryPlanToBe = compiler.compile();
        // steps and parallelIteratorFactory did not take effect.
        if (queryPlanToBe.getTableRef().getTable().getType() == PTableType.PROJECTED || queryPlanToBe.getTableRef().getTable().getType() == PTableType.SUBQUERY) {
            parallelIteratorFactoryToBe = null;
        }
        nValuesToSet = queryPlanToBe.getProjector().getColumnCount();
    // Cannot auto commit if doing aggregation or topN or salted
    // Salted causes problems because the row may end up living on a different region
    } else {
        nValuesToSet = valueNodes.size() + addViewColumnsToBe.size() + (isTenantSpecific ? 1 : 0) + (isSharedViewIndex ? 1 : 0);
    }
    // Resize down to allow a subset of columns to be specifiable
    if (columnNodes.isEmpty() && columnIndexesToBe.length >= nValuesToSet) {
        nColumnsToSet = nValuesToSet;
        columnIndexesToBe = Arrays.copyOf(columnIndexesToBe, nValuesToSet);
        pkSlotIndexesToBe = Arrays.copyOf(pkSlotIndexesToBe, nValuesToSet);
        for (int i = posOffset + nValuesToSet; i < table.getColumns().size(); i++) {
            PColumn column = table.getColumns().get(i);
            if (!column.isNullable() && column.getExpressionStr() == null) {
                throw new ConstraintViolationException(SchemaUtil.getColumnDisplayName(column) + " may not be null");
            }
        }
    }
    if (nValuesToSet != nColumnsToSet) {
        // been removed and the added back and we wouldn't detect that here.
        throw new UpsertColumnsValuesMismatchException(schemaName, tableName, "Numbers of columns: " + nColumnsToSet + ". Number of values: " + nValuesToSet);
    }
    final QueryPlan originalQueryPlan = queryPlanToBe;
    RowProjector projectorToBe = null;
    // Optimize only after all checks have been performed
    if (valueNodes == null) {
        queryPlanToBe = new QueryOptimizer(services).optimize(queryPlanToBe, statement, targetColumns, parallelIteratorFactoryToBe);
        projectorToBe = queryPlanToBe.getProjector();
    }
    final List<PColumn> allColumns = allColumnsToBe;
    final RowProjector projector = projectorToBe;
    final QueryPlan queryPlan = queryPlanToBe;
    final TableRef tableRef = tableRefToBe;
    final Set<PColumn> addViewColumns = addViewColumnsToBe;
    final Set<PColumn> overlapViewColumns = overlapViewColumnsToBe;
    final UpsertingParallelIteratorFactory parallelIteratorFactory = parallelIteratorFactoryToBe;
    final int[] columnIndexes = columnIndexesToBe;
    final int[] pkSlotIndexes = pkSlotIndexesToBe;
    final boolean useServerTimestamp = useServerTimestampToBe;
    if (table.getRowTimestampColPos() == -1 && useServerTimestamp) {
        throw new IllegalStateException("For a table without row timestamp column, useServerTimestamp cannot be true");
    }
    // ///////////////////////////////////////////////////////////////////
    if (valueNodes == null) {
        // Before we re-order, check that for updatable view columns
        // the projected expression either matches the column name or
        // is a constant with the same required value.
        throwIfNotUpdatable(tableRef, overlapViewColumnsToBe, targetColumns, projector, sameTable);
        // ///////////////////////////////////////////////////////////////////
        if (runOnServer) {
            // At most this array will grow bigger by the number of PK columns
            int[] allColumnsIndexes = Arrays.copyOf(columnIndexes, columnIndexes.length + nValuesToSet);
            int[] reverseColumnIndexes = new int[table.getColumns().size()];
            List<Expression> projectedExpressions = Lists.newArrayListWithExpectedSize(reverseColumnIndexes.length);
            Arrays.fill(reverseColumnIndexes, -1);
            for (int i = 0; i < nValuesToSet; i++) {
                projectedExpressions.add(projector.getColumnProjector(i).getExpression());
                reverseColumnIndexes[columnIndexes[i]] = i;
            }
            /*
                 * Order projected columns and projected expressions with PK columns
                 * leading order by slot position
                 */
            int offset = table.getBucketNum() == null ? 0 : 1;
            for (int i = 0; i < table.getPKColumns().size() - offset; i++) {
                PColumn column = table.getPKColumns().get(i + offset);
                int pos = reverseColumnIndexes[column.getPosition()];
                if (pos == -1) {
                    // it's not valid to set a fixed width type to null.
                    if (column.getDataType().isFixedWidth()) {
                        continue;
                    }
                    // Add literal null for missing PK columns
                    pos = projectedExpressions.size();
                    Expression literalNull = LiteralExpression.newConstant(null, column.getDataType(), Determinism.ALWAYS);
                    projectedExpressions.add(literalNull);
                    allColumnsIndexes[pos] = column.getPosition();
                }
                // Swap select expression at pos with i
                Collections.swap(projectedExpressions, i, pos);
                // Swap column indexes and reverse column indexes too
                int tempPos = allColumnsIndexes[i];
                allColumnsIndexes[i] = allColumnsIndexes[pos];
                allColumnsIndexes[pos] = tempPos;
                reverseColumnIndexes[tempPos] = pos;
                reverseColumnIndexes[i] = i;
            }
            // If any pk slots are changing and server side UPSERT SELECT is disabled, do not run on server
            if (!serverUpsertSelectEnabled && ExpressionUtil.isPkPositionChanging(new TableRef(table), projectedExpressions)) {
                runOnServer = false;
            }
            // ///////////////////////////////////////////////////////////////////
            if (runOnServer) {
                // Iterate through columns being projected
                List<PColumn> projectedColumns = Lists.newArrayListWithExpectedSize(projectedExpressions.size());
                int posOff = table.getBucketNum() != null ? 1 : 0;
                for (int i = 0; i < projectedExpressions.size(); i++) {
                    // Must make new column if position has changed
                    PColumn column = allColumns.get(allColumnsIndexes[i]);
                    projectedColumns.add(column.getPosition() == i + posOff ? column : new PColumnImpl(column, i + posOff));
                }
                // Build table from projectedColumns
                // Hack to add default column family to be used on server in case no value column is projected.
                PTable projectedTable = PTableImpl.makePTable(table, projectedColumns, PNameFactory.newName(SchemaUtil.getEmptyColumnFamily(table)));
                SelectStatement select = SelectStatement.create(SelectStatement.COUNT_ONE, upsert.getHint());
                StatementContext statementContext = queryPlan.getContext();
                RowProjector aggProjectorToBe = ProjectionCompiler.compile(statementContext, select, GroupBy.EMPTY_GROUP_BY);
                statementContext.getAggregationManager().compile(queryPlan.getContext(), GroupBy.EMPTY_GROUP_BY);
                if (queryPlan.getProjector().projectEveryRow()) {
                    aggProjectorToBe = new RowProjector(aggProjectorToBe, true);
                }
                final RowProjector aggProjector = aggProjectorToBe;
                /*
                     * Transfer over PTable representing subset of columns selected, but all PK columns.
                     * Move columns setting PK first in pkSlot order, adding LiteralExpression of null for any missing ones.
                     * Transfer over List<Expression> for projection.
                     * In region scan, evaluate expressions in order, collecting first n columns for PK and collection non PK in mutation Map
                     * Create the PRow and get the mutations, adding them to the batch
                     */
                final StatementContext context = queryPlan.getContext();
                final Scan scan = context.getScan();
                scan.setAttribute(BaseScannerRegionObserver.UPSERT_SELECT_TABLE, UngroupedAggregateRegionObserver.serialize(projectedTable));
                scan.setAttribute(BaseScannerRegionObserver.UPSERT_SELECT_EXPRS, UngroupedAggregateRegionObserver.serialize(projectedExpressions));
                // Ignore order by - it has no impact
                final QueryPlan aggPlan = new AggregatePlan(context, select, statementContext.getCurrentTable(), aggProjector, null, null, OrderBy.EMPTY_ORDER_BY, null, GroupBy.EMPTY_GROUP_BY, null, originalQueryPlan);
                return new ServerUpsertSelectMutationPlan(queryPlan, tableRef, originalQueryPlan, context, connection, scan, aggPlan, aggProjector, maxSize, maxSizeBytes);
            }
        }
        // ///////////////////////////////////////////////////////////////////
        return new ClientUpsertSelectMutationPlan(queryPlan, tableRef, originalQueryPlan, parallelIteratorFactory, projector, columnIndexes, pkSlotIndexes, useServerTimestamp, maxSize, maxSizeBytes);
    }
    // //////////////////////////////////////////////////////////////////
    // UPSERT VALUES
    // ///////////////////////////////////////////////////////////////////
    final byte[][] values = new byte[nValuesToSet][];
    int nodeIndex = 0;
    if (isSharedViewIndex) {
        values[nodeIndex++] = MetaDataUtil.getViewIndexIdDataType().toBytes(table.getViewIndexId());
    }
    if (isTenantSpecific) {
        PName tenantId = connection.getTenantId();
        values[nodeIndex++] = ScanUtil.getTenantIdBytes(table.getRowKeySchema(), table.getBucketNum() != null, tenantId, isSharedViewIndex);
    }
    final int nodeIndexOffset = nodeIndex;
    // Allocate array based on size of all columns in table,
    // since some values may not be set (if they're nullable).
    final StatementContext context = new StatementContext(statement, resolver, new Scan(), new SequenceManager(statement));
    UpsertValuesCompiler expressionBuilder = new UpsertValuesCompiler(context);
    final List<Expression> constantExpressions = Lists.newArrayListWithExpectedSize(valueNodes.size());
    // and initialize them in one batch
    for (ParseNode valueNode : valueNodes) {
        if (!valueNode.isStateless()) {
            throw new SQLExceptionInfo.Builder(SQLExceptionCode.VALUE_IN_UPSERT_NOT_CONSTANT).build().buildException();
        }
        PColumn column = allColumns.get(columnIndexes[nodeIndex]);
        expressionBuilder.setColumn(column);
        Expression expression = valueNode.accept(expressionBuilder);
        if (expression.getDataType() != null && !expression.getDataType().isCastableTo(column.getDataType())) {
            throw TypeMismatchException.newException(expression.getDataType(), column.getDataType(), "expression: " + expression.toString() + " in column " + column);
        }
        constantExpressions.add(expression);
        nodeIndex++;
    }
    byte[] onDupKeyBytesToBe = null;
    List<Pair<ColumnName, ParseNode>> onDupKeyPairs = upsert.getOnDupKeyPairs();
    if (onDupKeyPairs != null) {
        if (table.isImmutableRows()) {
            throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_USE_ON_DUP_KEY_FOR_IMMUTABLE).setSchemaName(table.getSchemaName().getString()).setTableName(table.getTableName().getString()).build().buildException();
        }
        if (table.isTransactional()) {
            throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_USE_ON_DUP_KEY_FOR_TRANSACTIONAL).setSchemaName(table.getSchemaName().getString()).setTableName(table.getTableName().getString()).build().buildException();
        }
        if (connection.getSCN() != null) {
            throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_SET_SCN_IN_ON_DUP_KEY).setSchemaName(table.getSchemaName().getString()).setTableName(table.getTableName().getString()).build().buildException();
        }
        if (SchemaUtil.hasGlobalIndex(table)) {
            throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_USE_ON_DUP_KEY_WITH_GLOBAL_IDX).setSchemaName(table.getSchemaName().getString()).setTableName(table.getTableName().getString()).build().buildException();
        }
        if (onDupKeyPairs.isEmpty()) {
            // ON DUPLICATE KEY IGNORE
            onDupKeyBytesToBe = PhoenixIndexBuilder.serializeOnDupKeyIgnore();
        } else {
            // ON DUPLICATE KEY UPDATE;
            int position = table.getBucketNum() == null ? 0 : 1;
            UpdateColumnCompiler compiler = new UpdateColumnCompiler(context);
            int nColumns = onDupKeyPairs.size();
            List<Expression> updateExpressions = Lists.newArrayListWithExpectedSize(nColumns);
            LinkedHashSet<PColumn> updateColumns = Sets.newLinkedHashSetWithExpectedSize(nColumns + 1);
            updateColumns.add(new PColumnImpl(// Use first PK column name as we know it won't conflict with others
            table.getPKColumns().get(position).getName(), null, PVarbinary.INSTANCE, null, null, false, position, SortOrder.getDefault(), 0, null, false, null, false, false, null));
            position++;
            for (Pair<ColumnName, ParseNode> columnPair : onDupKeyPairs) {
                ColumnName colName = columnPair.getFirst();
                PColumn updateColumn = resolver.resolveColumn(null, colName.getFamilyName(), colName.getColumnName()).getColumn();
                if (SchemaUtil.isPKColumn(updateColumn)) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_UPDATE_PK_ON_DUP_KEY).setSchemaName(table.getSchemaName().getString()).setTableName(table.getTableName().getString()).setColumnName(updateColumn.getName().getString()).build().buildException();
                }
                final int columnPosition = position++;
                if (!updateColumns.add(new DelegateColumn(updateColumn) {

                    @Override
                    public int getPosition() {
                        return columnPosition;
                    }
                })) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.DUPLICATE_COLUMN_IN_ON_DUP_KEY).setSchemaName(table.getSchemaName().getString()).setTableName(table.getTableName().getString()).setColumnName(updateColumn.getName().getString()).build().buildException();
                }
                ;
                ParseNode updateNode = columnPair.getSecond();
                compiler.setColumn(updateColumn);
                Expression updateExpression = updateNode.accept(compiler);
                // Check that updateExpression is coercible to updateColumn
                if (updateExpression.getDataType() != null && !updateExpression.getDataType().isCastableTo(updateColumn.getDataType())) {
                    throw TypeMismatchException.newException(updateExpression.getDataType(), updateColumn.getDataType(), "expression: " + updateExpression.toString() + " for column " + updateColumn);
                }
                if (compiler.isAggregate()) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.AGGREGATION_NOT_ALLOWED_IN_ON_DUP_KEY).setSchemaName(table.getSchemaName().getString()).setTableName(table.getTableName().getString()).setColumnName(updateColumn.getName().getString()).build().buildException();
                }
                updateExpressions.add(updateExpression);
            }
            PTable onDupKeyTable = PTableImpl.makePTable(table, updateColumns);
            onDupKeyBytesToBe = PhoenixIndexBuilder.serializeOnDupKeyUpdate(onDupKeyTable, updateExpressions);
        }
    }
    final byte[] onDupKeyBytes = onDupKeyBytesToBe;
    return new UpsertValuesMutationPlan(context, tableRef, nodeIndexOffset, constantExpressions, allColumns, columnIndexes, overlapViewColumns, values, addViewColumns, connection, pkSlotIndexes, useServerTimestamp, onDupKeyBytes, maxSize, maxSizeBytes);
}