Examples with RowKeyColumnExpression org.apache.phoenix.expression.RowKeyColumnExpression used on opensource projects

Example 6 with RowKeyColumnExpression

use of org.apache.phoenix.expression.RowKeyColumnExpression in project phoenix by apache.

the class MetaDataClient method createIndex.

/**
     * Create an index table by morphing the CreateIndexStatement into a CreateTableStatement and calling
     * MetaDataClient.createTable. In doing so, we perform the following translations:
     * 1) Change the type of any columns being indexed to types that support null if the column is nullable.
     *    For example, a BIGINT type would be coerced to a DECIMAL type, since a DECIMAL type supports null
     *    when it's in the row key while a BIGINT does not.
     * 2) Append any row key column from the data table that is not in the indexed column list. Our indexes
     *    rely on having a 1:1 correspondence between the index and data rows.
     * 3) Change the name of the columns to include the column family. For example, if you have a column
     *    named "B" in a column family named "A", the indexed column name will be "A:B". This makes it easy
     *    to translate the column references in a query to the correct column references in an index table
     *    regardless of whether the column reference is prefixed with the column family name or not. It also
     *    has the side benefit of allowing the same named column in different column families to both be
     *    listed as an index column.
     * @param statement
     * @param splits
     * @return MutationState from population of index table from data table
     * @throws SQLException
     */
public MutationState createIndex(CreateIndexStatement statement, byte[][] splits) throws SQLException {
    IndexKeyConstraint ik = statement.getIndexConstraint();
    TableName indexTableName = statement.getIndexTableName();
    Map<String, Object> tableProps = Maps.newHashMapWithExpectedSize(statement.getProps().size());
    Map<String, Object> commonFamilyProps = Maps.newHashMapWithExpectedSize(statement.getProps().size() + 1);
    populatePropertyMaps(statement.getProps(), tableProps, commonFamilyProps);
    List<Pair<ParseNode, SortOrder>> indexParseNodeAndSortOrderList = ik.getParseNodeAndSortOrderList();
    List<ColumnName> includedColumns = statement.getIncludeColumns();
    TableRef tableRef = null;
    PTable table = null;
    int numRetries = 0;
    boolean allocateIndexId = false;
    boolean isLocalIndex = statement.getIndexType() == IndexType.LOCAL;
    int hbaseVersion = connection.getQueryServices().getLowestClusterHBaseVersion();
    if (isLocalIndex) {
        if (!connection.getQueryServices().getProps().getBoolean(QueryServices.ALLOW_LOCAL_INDEX_ATTRIB, QueryServicesOptions.DEFAULT_ALLOW_LOCAL_INDEX)) {
            throw new SQLExceptionInfo.Builder(SQLExceptionCode.UNALLOWED_LOCAL_INDEXES).setTableName(indexTableName.getTableName()).build().buildException();
        }
        if (!connection.getQueryServices().supportsFeature(Feature.LOCAL_INDEX)) {
            throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_LOCAL_INDEXES).setTableName(indexTableName.getTableName()).build().buildException();
        }
    }
    while (true) {
        try {
            ColumnResolver resolver = FromCompiler.getResolver(statement, connection, statement.getUdfParseNodes());
            tableRef = resolver.getTables().get(0);
            Date asyncCreatedDate = null;
            if (statement.isAsync()) {
                asyncCreatedDate = new Date(tableRef.getTimeStamp());
            }
            PTable dataTable = tableRef.getTable();
            boolean isTenantConnection = connection.getTenantId() != null;
            if (isTenantConnection) {
                if (dataTable.getType() != PTableType.VIEW) {
                    throw new SQLFeatureNotSupportedException("An index may only be created for a VIEW through a tenant-specific connection");
                }
            }
            if (!dataTable.isImmutableRows()) {
                if (hbaseVersion < PhoenixDatabaseMetaData.MUTABLE_SI_VERSION_THRESHOLD) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES).setTableName(indexTableName.getTableName()).build().buildException();
                }
                if (!connection.getQueryServices().hasIndexWALCodec() && !dataTable.isTransactional()) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_MUTABLE_INDEX_CONFIG).setTableName(indexTableName.getTableName()).build().buildException();
                }
            }
            int posOffset = 0;
            List<PColumn> pkColumns = dataTable.getPKColumns();
            Set<RowKeyColumnExpression> unusedPkColumns;
            if (dataTable.getBucketNum() != null) {
                // Ignore SALT column
                unusedPkColumns = Sets.newLinkedHashSetWithExpectedSize(pkColumns.size() - 1);
                posOffset++;
            } else {
                unusedPkColumns = Sets.newLinkedHashSetWithExpectedSize(pkColumns.size());
            }
            for (int i = posOffset; i < pkColumns.size(); i++) {
                PColumn column = pkColumns.get(i);
                unusedPkColumns.add(new RowKeyColumnExpression(column, new RowKeyValueAccessor(pkColumns, i), "\"" + column.getName().getString() + "\""));
            }
            List<ColumnDefInPkConstraint> allPkColumns = Lists.newArrayListWithExpectedSize(unusedPkColumns.size());
            List<ColumnDef> columnDefs = Lists.newArrayListWithExpectedSize(includedColumns.size() + indexParseNodeAndSortOrderList.size());
            /*
                 * Allocate an index ID in two circumstances:
                 * 1) for a local index, as all local indexes will reside in the same HBase table
                 * 2) for a view on an index.
                 */
            if (isLocalIndex || (dataTable.getType() == PTableType.VIEW && dataTable.getViewType() != ViewType.MAPPED)) {
                allocateIndexId = true;
                PDataType dataType = MetaDataUtil.getViewIndexIdDataType();
                ColumnName colName = ColumnName.caseSensitiveColumnName(MetaDataUtil.getViewIndexIdColumnName());
                allPkColumns.add(new ColumnDefInPkConstraint(colName, SortOrder.getDefault(), false));
                columnDefs.add(FACTORY.columnDef(colName, dataType.getSqlTypeName(), false, null, null, false, SortOrder.getDefault(), null, false));
            }
            if (dataTable.isMultiTenant()) {
                PColumn col = dataTable.getPKColumns().get(posOffset);
                RowKeyColumnExpression columnExpression = new RowKeyColumnExpression(col, new RowKeyValueAccessor(pkColumns, posOffset), col.getName().getString());
                unusedPkColumns.remove(columnExpression);
                PDataType dataType = IndexUtil.getIndexColumnDataType(col);
                ColumnName colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(col));
                allPkColumns.add(new ColumnDefInPkConstraint(colName, col.getSortOrder(), false));
                columnDefs.add(FACTORY.columnDef(colName, dataType.getSqlTypeName(), col.isNullable(), col.getMaxLength(), col.getScale(), false, SortOrder.getDefault(), col.getName().getString(), col.isRowTimestamp()));
            }
            PhoenixStatement phoenixStatment = new PhoenixStatement(connection);
            StatementContext context = new StatementContext(phoenixStatment, resolver);
            IndexExpressionCompiler expressionIndexCompiler = new IndexExpressionCompiler(context);
            Set<ColumnName> indexedColumnNames = Sets.newHashSetWithExpectedSize(indexParseNodeAndSortOrderList.size());
            for (Pair<ParseNode, SortOrder> pair : indexParseNodeAndSortOrderList) {
                ParseNode parseNode = pair.getFirst();
                // normalize the parse node
                parseNode = StatementNormalizer.normalize(parseNode, resolver);
                // compile the parseNode to get an expression
                expressionIndexCompiler.reset();
                Expression expression = parseNode.accept(expressionIndexCompiler);
                if (expressionIndexCompiler.isAggregate()) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.AGGREGATE_EXPRESSION_NOT_ALLOWED_IN_INDEX).build().buildException();
                }
                if (expression.getDeterminism() != Determinism.ALWAYS) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.NON_DETERMINISTIC_EXPRESSION_NOT_ALLOWED_IN_INDEX).build().buildException();
                }
                if (expression.isStateless()) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.STATELESS_EXPRESSION_NOT_ALLOWED_IN_INDEX).build().buildException();
                }
                unusedPkColumns.remove(expression);
                // Go through parse node to get string as otherwise we
                // can lose information during compilation
                StringBuilder buf = new StringBuilder();
                parseNode.toSQL(resolver, buf);
                // need to escape backslash as this expression will be re-parsed later
                String expressionStr = StringUtil.escapeBackslash(buf.toString());
                ColumnName colName = null;
                ColumnRef colRef = expressionIndexCompiler.getColumnRef();
                boolean isRowTimestamp = false;
                if (colRef != null) {
                    // if this is a regular column
                    PColumn column = colRef.getColumn();
                    String columnFamilyName = column.getFamilyName() != null ? column.getFamilyName().getString() : null;
                    colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(columnFamilyName, column.getName().getString()));
                    isRowTimestamp = column.isRowTimestamp();
                    if (colRef.getColumn().getExpressionStr() != null) {
                        expressionStr = colRef.getColumn().getExpressionStr();
                    }
                } else {
                    // if this is an expression
                    // TODO column names cannot have double quotes, remove this once this PHOENIX-1621 is fixed
                    String name = expressionStr.replaceAll("\"", "'");
                    colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(null, name));
                }
                indexedColumnNames.add(colName);
                PDataType dataType = IndexUtil.getIndexColumnDataType(expression.isNullable(), expression.getDataType());
                allPkColumns.add(new ColumnDefInPkConstraint(colName, pair.getSecond(), isRowTimestamp));
                columnDefs.add(FACTORY.columnDef(colName, dataType.getSqlTypeName(), expression.isNullable(), expression.getMaxLength(), expression.getScale(), false, pair.getSecond(), expressionStr, isRowTimestamp));
            }
            // Next all the PK columns from the data table that aren't indexed
            if (!unusedPkColumns.isEmpty()) {
                for (RowKeyColumnExpression colExpression : unusedPkColumns) {
                    PColumn col = dataTable.getPKColumns().get(colExpression.getPosition());
                    // we don't need these in the index
                    if (col.getViewConstant() == null) {
                        ColumnName colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(col));
                        allPkColumns.add(new ColumnDefInPkConstraint(colName, colExpression.getSortOrder(), col.isRowTimestamp()));
                        PDataType dataType = IndexUtil.getIndexColumnDataType(colExpression.isNullable(), colExpression.getDataType());
                        columnDefs.add(FACTORY.columnDef(colName, dataType.getSqlTypeName(), colExpression.isNullable(), colExpression.getMaxLength(), colExpression.getScale(), false, colExpression.getSortOrder(), colExpression.toString(), col.isRowTimestamp()));
                    }
                }
            }
            // Last all the included columns (minus any PK columns)
            for (ColumnName colName : includedColumns) {
                PColumn col = resolver.resolveColumn(null, colName.getFamilyName(), colName.getColumnName()).getColumn();
                colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(col));
                // Check for duplicates between indexed and included columns
                if (indexedColumnNames.contains(colName)) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.COLUMN_EXIST_IN_DEF).build().buildException();
                }
                if (!SchemaUtil.isPKColumn(col) && col.getViewConstant() == null) {
                    // Need to re-create ColumnName, since the above one won't have the column family name
                    colName = ColumnName.caseSensitiveColumnName(isLocalIndex ? IndexUtil.getLocalIndexColumnFamily(col.getFamilyName().getString()) : col.getFamilyName().getString(), IndexUtil.getIndexColumnName(col));
                    columnDefs.add(FACTORY.columnDef(colName, col.getDataType().getSqlTypeName(), col.isNullable(), col.getMaxLength(), col.getScale(), false, col.getSortOrder(), col.getExpressionStr(), col.isRowTimestamp()));
                }
            }
            // We need this in the props so that the correct column family is created
            if (dataTable.getDefaultFamilyName() != null && dataTable.getType() != PTableType.VIEW && !allocateIndexId) {
                statement.getProps().put("", new Pair<String, Object>(DEFAULT_COLUMN_FAMILY_NAME, dataTable.getDefaultFamilyName().getString()));
            }
            PrimaryKeyConstraint pk = FACTORY.primaryKey(null, allPkColumns);
            tableProps.put(MetaDataUtil.DATA_TABLE_NAME_PROP_NAME, dataTable.getName().getString());
            CreateTableStatement tableStatement = FACTORY.createTable(indexTableName, statement.getProps(), columnDefs, pk, statement.getSplitNodes(), PTableType.INDEX, statement.ifNotExists(), null, null, statement.getBindCount(), null);
            table = createTableInternal(tableStatement, splits, dataTable, null, null, null, null, allocateIndexId, statement.getIndexType(), asyncCreatedDate, tableProps, commonFamilyProps);
            break;
        } catch (ConcurrentTableMutationException e) {
            // Can happen if parent data table changes while above is in progress
            if (numRetries < 5) {
                numRetries++;
                continue;
            }
            throw e;
        }
    }
    if (table == null) {
        return new MutationState(0, 0, connection);
    }
    if (logger.isInfoEnabled())
        logger.info("Created index " + table.getName().getString() + " at " + table.getTimeStamp());
    boolean asyncIndexBuildEnabled = connection.getQueryServices().getProps().getBoolean(QueryServices.INDEX_ASYNC_BUILD_ENABLED, QueryServicesOptions.DEFAULT_INDEX_ASYNC_BUILD_ENABLED);
    // In async process, we return immediately as the MR job needs to be triggered .
    if (statement.isAsync() && asyncIndexBuildEnabled) {
        return new MutationState(0, 0, connection);
    }
    // connection so that our new index table is visible.
    if (connection.getSCN() != null) {
        return buildIndexAtTimeStamp(table, statement.getTable());
    }
    return buildIndex(table, tableRef);
}

Example 7 with RowKeyColumnExpression

use of org.apache.phoenix.expression.RowKeyColumnExpression in project phoenix by apache.

the class MetaDataClient method addColumn.

public MutationState addColumn(PTable table, List<ColumnDef> origColumnDefs, ListMultimap<String, Pair<String, Object>> stmtProperties, boolean ifNotExists, boolean removeTableProps, NamedTableNode namedTableNode, PTableType tableType) throws SQLException {
    connection.rollback();
    boolean wasAutoCommit = connection.getAutoCommit();
    try {
        connection.setAutoCommit(false);
        PName tenantId = connection.getTenantId();
        String schemaName = table.getSchemaName().getString();
        String tableName = table.getTableName().getString();
        Boolean isImmutableRowsProp = null;
        Boolean multiTenantProp = null;
        Boolean disableWALProp = null;
        Boolean storeNullsProp = null;
        Boolean isTransactionalProp = null;
        Long updateCacheFrequencyProp = null;
        Boolean appendOnlySchemaProp = null;
        Long guidePostWidth = -1L;
        ImmutableStorageScheme immutableStorageSchemeProp = null;
        Boolean useStatsForParallelizationProp = null;
        Map<String, List<Pair<String, Object>>> properties = new HashMap<>(stmtProperties.size());
        List<ColumnDef> columnDefs = null;
        if (table.isAppendOnlySchema()) {
            // only make the rpc if we are adding new columns
            columnDefs = Lists.newArrayList();
            for (ColumnDef columnDef : origColumnDefs) {
                String familyName = columnDef.getColumnDefName().getFamilyName();
                String columnName = columnDef.getColumnDefName().getColumnName();
                if (familyName != null) {
                    try {
                        PColumnFamily columnFamily = table.getColumnFamily(familyName);
                        columnFamily.getPColumnForColumnName(columnName);
                        if (!ifNotExists) {
                            throw new ColumnAlreadyExistsException(schemaName, tableName, columnName);
                        }
                    } catch (ColumnFamilyNotFoundException | ColumnNotFoundException e) {
                        columnDefs.add(columnDef);
                    }
                } else {
                    try {
                        table.getColumnForColumnName(columnName);
                        if (!ifNotExists) {
                            throw new ColumnAlreadyExistsException(schemaName, tableName, columnName);
                        }
                    } catch (ColumnNotFoundException e) {
                        columnDefs.add(columnDef);
                    }
                }
            }
        } else {
            columnDefs = origColumnDefs == null ? Collections.<ColumnDef>emptyList() : origColumnDefs;
        }
        for (String family : stmtProperties.keySet()) {
            List<Pair<String, Object>> origPropsList = stmtProperties.get(family);
            List<Pair<String, Object>> propsList = Lists.newArrayListWithExpectedSize(origPropsList.size());
            for (Pair<String, Object> prop : origPropsList) {
                String propName = prop.getFirst();
                if (TableProperty.isPhoenixTableProperty(propName)) {
                    TableProperty tableProp = TableProperty.valueOf(propName);
                    tableProp.validate(true, !family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY), table.getType());
                    Object value = tableProp.getValue(prop.getSecond());
                    if (propName.equals(PTable.IS_IMMUTABLE_ROWS_PROP_NAME)) {
                        isImmutableRowsProp = (Boolean) value;
                    } else if (propName.equals(PhoenixDatabaseMetaData.MULTI_TENANT)) {
                        multiTenantProp = (Boolean) value;
                    } else if (propName.equals(DISABLE_WAL)) {
                        disableWALProp = (Boolean) value;
                    } else if (propName.equals(STORE_NULLS)) {
                        storeNullsProp = (Boolean) value;
                    } else if (propName.equals(TRANSACTIONAL)) {
                        isTransactionalProp = (Boolean) value;
                    } else if (propName.equals(UPDATE_CACHE_FREQUENCY)) {
                        updateCacheFrequencyProp = (Long) value;
                    } else if (propName.equals(GUIDE_POSTS_WIDTH)) {
                        guidePostWidth = (Long) value;
                    } else if (propName.equals(APPEND_ONLY_SCHEMA)) {
                        appendOnlySchemaProp = (Boolean) value;
                    } else if (propName.equalsIgnoreCase(IMMUTABLE_STORAGE_SCHEME)) {
                        immutableStorageSchemeProp = (ImmutableStorageScheme) value;
                    } else if (propName.equalsIgnoreCase(USE_STATS_FOR_PARALLELIZATION)) {
                        useStatsForParallelizationProp = (Boolean) value;
                    }
                }
                // if removeTableProps is true only add the property if it is not a HTable or Phoenix Table property
                if (!removeTableProps || (!TableProperty.isPhoenixTableProperty(propName) && !MetaDataUtil.isHTableProperty(propName))) {
                    propsList.add(prop);
                }
            }
            properties.put(family, propsList);
        }
        boolean retried = false;
        boolean changingPhoenixTableProperty = false;
        boolean nonTxToTx = false;
        while (true) {
            ColumnResolver resolver = FromCompiler.getResolver(namedTableNode, connection);
            table = resolver.getTables().get(0).getTable();
            int nIndexes = table.getIndexes().size();
            int numCols = columnDefs.size();
            int nNewColumns = numCols;
            List<Mutation> tableMetaData = Lists.newArrayListWithExpectedSize((1 + nNewColumns) * (nIndexes + 1));
            List<Mutation> columnMetaData = Lists.newArrayListWithExpectedSize(nNewColumns * (nIndexes + 1));
            if (logger.isDebugEnabled()) {
                logger.debug(LogUtil.addCustomAnnotations("Resolved table to " + table.getName().getString() + " with seqNum " + table.getSequenceNumber() + " at timestamp " + table.getTimeStamp() + " with " + table.getColumns().size() + " columns: " + table.getColumns(), connection));
            }
            int position = table.getColumns().size();
            List<PColumn> currentPKs = table.getPKColumns();
            PColumn lastPK = currentPKs.get(currentPKs.size() - 1);
            // Disallow adding columns if the last column is VARBIANRY.
            if (lastPK.getDataType() == PVarbinary.INSTANCE || lastPK.getDataType().isArrayType()) {
                throw new SQLExceptionInfo.Builder(SQLExceptionCode.VARBINARY_LAST_PK).setColumnName(lastPK.getName().getString()).build().buildException();
            }
            // Disallow adding columns if last column is fixed width and nullable.
            if (lastPK.isNullable() && lastPK.getDataType().isFixedWidth()) {
                throw new SQLExceptionInfo.Builder(SQLExceptionCode.NULLABLE_FIXED_WIDTH_LAST_PK).setColumnName(lastPK.getName().getString()).build().buildException();
            }
            Boolean isImmutableRows = null;
            if (isImmutableRowsProp != null) {
                if (isImmutableRowsProp.booleanValue() != table.isImmutableRows()) {
                    if (table.getImmutableStorageScheme() != ImmutableStorageScheme.ONE_CELL_PER_COLUMN) {
                        throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_IMMUTABLE_ROWS_PROPERTY).setSchemaName(schemaName).setTableName(tableName).build().buildException();
                    }
                    isImmutableRows = isImmutableRowsProp;
                    changingPhoenixTableProperty = true;
                }
            }
            Boolean multiTenant = null;
            if (multiTenantProp != null) {
                if (multiTenantProp.booleanValue() != table.isMultiTenant()) {
                    multiTenant = multiTenantProp;
                    changingPhoenixTableProperty = true;
                }
            }
            Boolean disableWAL = null;
            if (disableWALProp != null) {
                if (disableWALProp.booleanValue() != table.isWALDisabled()) {
                    disableWAL = disableWALProp;
                    changingPhoenixTableProperty = true;
                }
            }
            Long updateCacheFrequency = null;
            if (updateCacheFrequencyProp != null) {
                if (updateCacheFrequencyProp.longValue() != table.getUpdateCacheFrequency()) {
                    updateCacheFrequency = updateCacheFrequencyProp;
                    changingPhoenixTableProperty = true;
                }
            }
            Boolean appendOnlySchema = null;
            if (appendOnlySchemaProp != null) {
                if (appendOnlySchemaProp != table.isAppendOnlySchema()) {
                    appendOnlySchema = appendOnlySchemaProp;
                    changingPhoenixTableProperty = true;
                }
            }
            ImmutableStorageScheme immutableStorageScheme = null;
            if (immutableStorageSchemeProp != null) {
                if (table.getImmutableStorageScheme() == ONE_CELL_PER_COLUMN || immutableStorageSchemeProp == ONE_CELL_PER_COLUMN) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE).setSchemaName(schemaName).setTableName(tableName).build().buildException();
                } else if (immutableStorageSchemeProp != table.getImmutableStorageScheme()) {
                    immutableStorageScheme = immutableStorageSchemeProp;
                    changingPhoenixTableProperty = true;
                }
            }
            if (guidePostWidth == null || guidePostWidth >= 0) {
                changingPhoenixTableProperty = true;
            }
            Boolean storeNulls = null;
            if (storeNullsProp != null) {
                if (storeNullsProp.booleanValue() != table.getStoreNulls()) {
                    storeNulls = storeNullsProp;
                    changingPhoenixTableProperty = true;
                }
            }
            Boolean useStatsForParallelization = null;
            if (useStatsForParallelizationProp != null) {
                if (useStatsForParallelizationProp.booleanValue() != table.useStatsForParallelization()) {
                    useStatsForParallelization = useStatsForParallelizationProp;
                    changingPhoenixTableProperty = true;
                }
            }
            Boolean isTransactional = null;
            if (isTransactionalProp != null) {
                if (isTransactionalProp.booleanValue() != table.isTransactional()) {
                    isTransactional = isTransactionalProp;
                    // delete markers.
                    if (!isTransactional) {
                        throw new SQLExceptionInfo.Builder(SQLExceptionCode.TX_MAY_NOT_SWITCH_TO_NON_TX).setSchemaName(schemaName).setTableName(tableName).build().buildException();
                    }
                    // cannot create a transactional table if transactions are disabled
                    boolean transactionsEnabled = connection.getQueryServices().getProps().getBoolean(QueryServices.TRANSACTIONS_ENABLED, QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED);
                    if (!transactionsEnabled) {
                        throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED).setSchemaName(schemaName).setTableName(tableName).build().buildException();
                    }
                    // cannot make a table transactional if it has a row timestamp column
                    if (SchemaUtil.hasRowTimestampColumn(table)) {
                        throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP).setSchemaName(schemaName).setTableName(tableName).build().buildException();
                    }
                    changingPhoenixTableProperty = true;
                    nonTxToTx = true;
                }
            }
            Long timeStamp = TransactionUtil.getTableTimestamp(connection, table.isTransactional() || nonTxToTx);
            int numPkColumnsAdded = 0;
            List<PColumn> columns = Lists.newArrayListWithExpectedSize(numCols);
            Set<String> colFamiliesForPColumnsToBeAdded = new LinkedHashSet<>();
            Set<String> families = new LinkedHashSet<>();
            PTable tableForCQCounters = tableType == PTableType.VIEW ? PhoenixRuntime.getTable(connection, table.getPhysicalName().getString()) : table;
            ;
            EncodedCQCounter cqCounterToUse = tableForCQCounters.getEncodedCQCounter();
            Map<String, Integer> changedCqCounters = new HashMap<>(numCols);
            if (numCols > 0) {
                StatementContext context = new StatementContext(new PhoenixStatement(connection), resolver);
                String addColumnSqlToUse = connection.isRunningUpgrade() && tableName.equals(PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE) && schemaName.equals(PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA) ? ALTER_SYSCATALOG_TABLE_UPGRADE : INSERT_COLUMN_ALTER_TABLE;
                try (PreparedStatement colUpsert = connection.prepareStatement(addColumnSqlToUse)) {
                    short nextKeySeq = SchemaUtil.getMaxKeySeq(table);
                    for (ColumnDef colDef : columnDefs) {
                        if (colDef != null && !colDef.isNull()) {
                            if (colDef.isPK()) {
                                throw new SQLExceptionInfo.Builder(SQLExceptionCode.NOT_NULLABLE_COLUMN_IN_ROW_KEY).setColumnName(colDef.getColumnDefName().getColumnName()).build().buildException();
                            } else {
                                throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ADD_NOT_NULLABLE_COLUMN).setColumnName(colDef.getColumnDefName().getColumnName()).build().buildException();
                            }
                        }
                        if (colDef != null && colDef.isPK() && table.getType() == VIEW && table.getViewType() != MAPPED) {
                            throwIfLastPKOfParentIsFixedLength(getParentOfView(table), schemaName, tableName, colDef);
                        }
                        if (colDef != null && colDef.isRowTimestamp()) {
                            throw new SQLExceptionInfo.Builder(SQLExceptionCode.ROWTIMESTAMP_CREATE_ONLY).setColumnName(colDef.getColumnDefName().getColumnName()).build().buildException();
                        }
                        if (!colDef.validateDefault(context, null)) {
                            // Remove DEFAULT as it's not necessary
                            colDef = new ColumnDef(colDef, null);
                        }
                        Integer encodedCQ = null;
                        if (!colDef.isPK()) {
                            String colDefFamily = colDef.getColumnDefName().getFamilyName();
                            String familyName = null;
                            ImmutableStorageScheme storageScheme = table.getImmutableStorageScheme();
                            String defaultColumnFamily = tableForCQCounters.getDefaultFamilyName() != null && !Strings.isNullOrEmpty(tableForCQCounters.getDefaultFamilyName().getString()) ? tableForCQCounters.getDefaultFamilyName().getString() : DEFAULT_COLUMN_FAMILY;
                            if (table.getType() == PTableType.INDEX && table.getIndexType() == IndexType.LOCAL) {
                                defaultColumnFamily = QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX + defaultColumnFamily;
                            }
                            if (storageScheme == SINGLE_CELL_ARRAY_WITH_OFFSETS) {
                                familyName = colDefFamily != null ? colDefFamily : defaultColumnFamily;
                            } else {
                                familyName = defaultColumnFamily;
                            }
                            encodedCQ = cqCounterToUse.getNextQualifier(familyName);
                            if (cqCounterToUse.increment(familyName)) {
                                changedCqCounters.put(familyName, cqCounterToUse.getNextQualifier(familyName));
                            }
                        }
                        byte[] columnQualifierBytes = null;
                        try {
                            columnQualifierBytes = EncodedColumnsUtil.getColumnQualifierBytes(colDef.getColumnDefName().getColumnName(), encodedCQ, table, colDef.isPK());
                        } catch (QualifierOutOfRangeException e) {
                            throw new SQLExceptionInfo.Builder(SQLExceptionCode.MAX_COLUMNS_EXCEEDED).setSchemaName(schemaName).setTableName(tableName).build().buildException();
                        }
                        PColumn column = newColumn(position++, colDef, PrimaryKeyConstraint.EMPTY, table.getDefaultFamilyName() == null ? null : table.getDefaultFamilyName().getString(), true, columnQualifierBytes);
                        columns.add(column);
                        String pkName = null;
                        Short keySeq = null;
                        // TODO: support setting properties on other families?
                        if (column.getFamilyName() == null) {
                            ++numPkColumnsAdded;
                            pkName = table.getPKName() == null ? null : table.getPKName().getString();
                            keySeq = ++nextKeySeq;
                        } else {
                            families.add(column.getFamilyName().getString());
                        }
                        colFamiliesForPColumnsToBeAdded.add(column.getFamilyName() == null ? null : column.getFamilyName().getString());
                        addColumnMutation(schemaName, tableName, column, colUpsert, null, pkName, keySeq, table.getBucketNum() != null);
                    }
                    // Add any new PK columns to end of index PK
                    if (numPkColumnsAdded > 0) {
                        // create PK column list that includes the newly created columns
                        List<PColumn> pkColumns = Lists.newArrayListWithExpectedSize(table.getPKColumns().size() + numPkColumnsAdded);
                        pkColumns.addAll(table.getPKColumns());
                        for (int i = 0; i < numCols; ++i) {
                            if (columnDefs.get(i).isPK()) {
                                pkColumns.add(columns.get(i));
                            }
                        }
                        int pkSlotPosition = table.getPKColumns().size() - 1;
                        for (PTable index : table.getIndexes()) {
                            short nextIndexKeySeq = SchemaUtil.getMaxKeySeq(index);
                            int indexPosition = index.getColumns().size();
                            for (int i = 0; i < numCols; ++i) {
                                ColumnDef colDef = columnDefs.get(i);
                                if (colDef.isPK()) {
                                    PDataType indexColDataType = IndexUtil.getIndexColumnDataType(colDef.isNull(), colDef.getDataType());
                                    ColumnName indexColName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(null, colDef.getColumnDefName().getColumnName()));
                                    Expression expression = new RowKeyColumnExpression(columns.get(i), new RowKeyValueAccessor(pkColumns, ++pkSlotPosition));
                                    ColumnDef indexColDef = FACTORY.columnDef(indexColName, indexColDataType.getSqlTypeName(), colDef.isNull(), colDef.getMaxLength(), colDef.getScale(), true, colDef.getSortOrder(), expression.toString(), colDef.isRowTimestamp());
                                    PColumn indexColumn = newColumn(indexPosition++, indexColDef, PrimaryKeyConstraint.EMPTY, null, true, null);
                                    addColumnMutation(schemaName, index.getTableName().getString(), indexColumn, colUpsert, index.getParentTableName().getString(), index.getPKName() == null ? null : index.getPKName().getString(), ++nextIndexKeySeq, index.getBucketNum() != null);
                                }
                            }
                        }
                    }
                    columnMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
                    connection.rollback();
                }
            } else {
                // have existing indexes.
                if (Boolean.FALSE.equals(isImmutableRows) && !table.getIndexes().isEmpty()) {
                    int hbaseVersion = connection.getQueryServices().getLowestClusterHBaseVersion();
                    if (hbaseVersion < PhoenixDatabaseMetaData.MUTABLE_SI_VERSION_THRESHOLD) {
                        throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES).setSchemaName(schemaName).setTableName(tableName).build().buildException();
                    }
                    if (!connection.getQueryServices().hasIndexWALCodec() && !table.isTransactional()) {
                        throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_MUTABLE_INDEX_CONFIG).setSchemaName(schemaName).setTableName(tableName).build().buildException();
                    }
                }
                if (Boolean.TRUE.equals(multiTenant)) {
                    throwIfInsufficientColumns(schemaName, tableName, table.getPKColumns(), table.getBucketNum() != null, multiTenant);
                }
            }
            if (!table.getIndexes().isEmpty() && (numPkColumnsAdded > 0 || nonTxToTx)) {
                for (PTable index : table.getIndexes()) {
                    incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, nonTxToTx ? Boolean.TRUE : null, updateCacheFrequency);
                }
                tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
                connection.rollback();
            }
            if (changingPhoenixTableProperty || columnDefs.size() > 0) {
                incrementTableSeqNum(table, tableType, columnDefs.size(), isTransactional, updateCacheFrequency, isImmutableRows, disableWAL, multiTenant, storeNulls, guidePostWidth, appendOnlySchema, immutableStorageScheme, useStatsForParallelization);
                tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
                connection.rollback();
            }
            // Force the table header row to be first
            Collections.reverse(tableMetaData);
            // Add column metadata afterwards, maintaining the order so columns have more predictable ordinal position
            tableMetaData.addAll(columnMetaData);
            boolean sharedIndex = tableType == PTableType.INDEX && (table.getIndexType() == IndexType.LOCAL || table.getViewIndexId() != null);
            String tenantIdToUse = connection.getTenantId() != null && sharedIndex ? connection.getTenantId().getString() : null;
            if (!changedCqCounters.isEmpty()) {
                PreparedStatement linkStatement;
                linkStatement = connection.prepareStatement(UPDATE_ENCODED_COLUMN_COUNTER);
                for (Entry<String, Integer> entry : changedCqCounters.entrySet()) {
                    linkStatement.setString(1, tenantIdToUse);
                    linkStatement.setString(2, tableForCQCounters.getSchemaName().getString());
                    linkStatement.setString(3, tableForCQCounters.getTableName().getString());
                    linkStatement.setString(4, entry.getKey());
                    linkStatement.setInt(5, entry.getValue());
                    linkStatement.execute();
                }
                // too since we want clients to get the latest PTable of the base table.
                if (tableType == VIEW) {
                    PreparedStatement incrementStatement = connection.prepareStatement(INCREMENT_SEQ_NUM);
                    incrementStatement.setString(1, null);
                    incrementStatement.setString(2, tableForCQCounters.getSchemaName().getString());
                    incrementStatement.setString(3, tableForCQCounters.getTableName().getString());
                    incrementStatement.setLong(4, tableForCQCounters.getSequenceNumber() + 1);
                    incrementStatement.execute();
                }
                tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
                connection.rollback();
            }
            byte[] family = families.size() > 0 ? families.iterator().next().getBytes() : null;
            // Figure out if the empty column family is changing as a result of adding the new column
            byte[] emptyCF = null;
            byte[] projectCF = null;
            if (table.getType() != PTableType.VIEW && family != null) {
                if (table.getColumnFamilies().isEmpty()) {
                    emptyCF = family;
                } else {
                    try {
                        table.getColumnFamily(family);
                    } catch (ColumnFamilyNotFoundException e) {
                        projectCF = family;
                        emptyCF = SchemaUtil.getEmptyColumnFamily(table);
                    }
                }
            }
            MetaDataMutationResult result = connection.getQueryServices().addColumn(tableMetaData, table, properties, colFamiliesForPColumnsToBeAdded, columns);
            try {
                MutationCode code = processMutationResult(schemaName, tableName, result);
                if (code == MutationCode.COLUMN_ALREADY_EXISTS) {
                    addTableToCache(result);
                    if (!ifNotExists) {
                        throw new ColumnAlreadyExistsException(schemaName, tableName, SchemaUtil.findExistingColumn(result.getTable(), columns));
                    }
                    return new MutationState(0, 0, connection);
                }
                // Only update client side cache if we aren't adding a PK column to a table with indexes or
                // transitioning a table from non transactional to transactional.
                // We could update the cache manually then too, it'd just be a pain.
                String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
                long resolvedTimeStamp = TransactionUtil.getResolvedTime(connection, result);
                if (table.getIndexes().isEmpty() || (numPkColumnsAdded == 0 && !nonTxToTx)) {
                    connection.addTable(result.getTable(), resolvedTimeStamp);
                    table = result.getTable();
                } else if (updateCacheFrequency != null) {
                    // Force removal from cache as the update cache frequency has changed
                    // Note that clients outside this JVM won't be affected.
                    connection.removeTable(tenantId, fullTableName, null, resolvedTimeStamp);
                }
                // We only need to do this if the multiTenant transitioned to false
                if (table.getType() == PTableType.TABLE && Boolean.FALSE.equals(multiTenant) && MetaDataUtil.hasViewIndexTable(connection, table.getPhysicalName())) {
                    connection.setAutoCommit(true);
                    MetaDataUtil.deleteViewIndexSequences(connection, table.getPhysicalName(), table.isNamespaceMapped());
                    // commands are run would remove all rows already.
                    if (!connection.getQueryServices().getProps().getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA)) {
                        Long scn = connection.getSCN();
                        long ts = (scn == null ? result.getMutationTime() : scn);
                        byte[] viewIndexPhysicalName = MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes());
                        PTable viewIndexTable = new PTableImpl(null, SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName), SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName), ts, table.getColumnFamilies(), table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.useStatsForParallelization());
                        List<TableRef> tableRefs = Collections.singletonList(new TableRef(null, viewIndexTable, ts, false));
                        MutationPlan plan = new PostDDLCompiler(connection).compile(tableRefs, null, null, Collections.<PColumn>emptyList(), ts);
                        connection.getQueryServices().updateData(plan);
                    }
                }
                if (emptyCF != null) {
                    Long scn = connection.getSCN();
                    connection.setAutoCommit(true);
                    // Delete everything in the column. You'll still be able to do queries at earlier timestamps
                    long ts = (scn == null ? result.getMutationTime() : scn);
                    MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, projectCF == null ? null : Collections.singletonList(projectCF), null, ts);
                    return connection.getQueryServices().updateData(plan);
                }
                return new MutationState(0, 0, connection);
            } catch (ConcurrentTableMutationException e) {
                if (retried) {
                    throw e;
                }
                if (logger.isDebugEnabled()) {
                    logger.debug(LogUtil.addCustomAnnotations("Caught ConcurrentTableMutationException for table " + SchemaUtil.getTableName(schemaName, tableName) + ". Will try again...", connection));
                }
                retried = true;
            }
        }
    } finally {
        connection.setAutoCommit(wasAutoCommit);
    }
}

Example 8 with RowKeyColumnExpression

use of org.apache.phoenix.expression.RowKeyColumnExpression in project phoenix by apache.

the class HavingCompilerTest method testOrAggFuncInHaving.

@Test
public void testOrAggFuncInHaving() throws SQLException {
    String query = "select count(1) from atable group by a_string having count(1) >= 1 or a_string = 'foo'";
    List<Object> binds = Collections.emptyList();
    Expressions expressions = compileStatement(query, binds);
    PColumn aCol = ATABLE.getColumnForColumnName("A_STRING");
    Expression h = or(constantComparison(CompareOp.GREATER_OR_EQUAL, new CountAggregateFunction(), 1L), constantComparison(CompareOp.EQUAL, new // a_string comes from group by key in this case
    RowKeyColumnExpression(// a_string comes from group by key in this case
    aCol, new RowKeyValueAccessor(Arrays.<PColumn>asList(aCol), 0)), "foo"));
    assertNull(expressions.whereClause);
    assertEquals(h, expressions.havingClause);
}

Example 9 with RowKeyColumnExpression

use of org.apache.phoenix.expression.RowKeyColumnExpression in project phoenix by apache.

the class ExpressionCompiler method wrapGroupByExpression.

private Expression wrapGroupByExpression(Expression expression) {
    // column.
    if (aggregateFunction == null) {
        int index = groupBy.getExpressions().indexOf(expression);
        if (index >= 0) {
            isAggregate = true;
            RowKeyValueAccessor accessor = new RowKeyValueAccessor(groupBy.getKeyExpressions(), index);
            expression = new RowKeyColumnExpression(expression, accessor, groupBy.getKeyExpressions().get(index).getDataType());
        }
    }
    return expression;
}

Example 10 with RowKeyColumnExpression

use of org.apache.phoenix.expression.RowKeyColumnExpression in project phoenix by apache.

the class WhereCompilerTest method testOrPKWithAndPKAndNotPK.

@Test
public void testOrPKWithAndPKAndNotPK() throws SQLException {
    String query = "select * from bugTable where ID = 'i1' or (ID = 'i2' and company = 'c3')";
    PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class);
    pconn.createStatement().execute("create table bugTable(ID varchar primary key,company varchar)");
    PhoenixPreparedStatement pstmt = newPreparedStatement(pconn, query);
    QueryPlan plan = pstmt.optimizeQuery();
    Scan scan = plan.getContext().getScan();
    Filter filter = scan.getFilter();
    Expression idExpression = new ColumnRef(plan.getTableRef(), plan.getTableRef().getTable().getColumnForColumnName("ID").getPosition()).newColumnExpression();
    Expression id = new RowKeyColumnExpression(idExpression, new RowKeyValueAccessor(plan.getTableRef().getTable().getPKColumns(), 0));
    Expression company = new KeyValueColumnExpression(plan.getTableRef().getTable().getColumnForColumnName("COMPANY"));
    // FilterList has no equals implementation
    assertTrue(filter instanceof FilterList);
    FilterList filterList = (FilterList) filter;
    assertEquals(FilterList.Operator.MUST_PASS_ALL, filterList.getOperator());
    assertEquals(Arrays.asList(new SkipScanFilter(ImmutableList.of(Arrays.asList(pointRange("i1"), pointRange("i2"))), SchemaUtil.VAR_BINARY_SCHEMA), singleKVFilter(or(constantComparison(CompareOp.EQUAL, id, "i1"), and(constantComparison(CompareOp.EQUAL, id, "i2"), constantComparison(CompareOp.EQUAL, company, "c3"))))), filterList.getFilters());
}