Examples with ColumnReference org.apache.phoenix.hbase.index.covered.update.ColumnReference used on opensource projects

Example 6 with ColumnReference

use of org.apache.phoenix.hbase.index.covered.update.ColumnReference in project phoenix by apache.

the class PhoenixIndexCodec method getIndexDeletes.

@Override
public Iterable<IndexUpdate> getIndexDeletes(TableState state, IndexMetaData context) throws IOException {
    PhoenixIndexMetaData metaData = (PhoenixIndexMetaData) context;
    List<IndexMaintainer> indexMaintainers = metaData.getIndexMaintainers();
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    ptr.set(state.getCurrentRowKey());
    List<IndexUpdate> indexUpdates = Lists.newArrayList();
    for (IndexMaintainer maintainer : indexMaintainers) {
        // For transactional tables, we use an index maintainer
        // to aid in rollback if there's a KeyValue column in the index. The alternative would be
        // to hold on to all uncommitted index row keys (even ones already sent to HBase) on the
        // client side.
        Set<ColumnReference> cols = Sets.newHashSet(maintainer.getAllColumns());
        cols.add(new ColumnReference(indexMaintainers.get(0).getDataEmptyKeyValueCF(), indexMaintainers.get(0).getEmptyKeyValueQualifier()));
        Pair<ValueGetter, IndexUpdate> statePair = state.getIndexUpdateState(cols, metaData.ignoreNewerMutations(), true, context);
        ValueGetter valueGetter = statePair.getFirst();
        if (valueGetter != null) {
            IndexUpdate indexUpdate = statePair.getSecond();
            indexUpdate.setTable(maintainer.isLocalIndex() ? state.getEnvironment().getRegion().getTableDesc().getName() : maintainer.getIndexTableName());
            Delete delete = maintainer.buildDeleteMutation(KV_BUILDER, valueGetter, ptr, state.getPendingUpdate(), state.getCurrentTimestamp(), env.getRegion().getRegionInfo().getStartKey(), env.getRegion().getRegionInfo().getEndKey());
            indexUpdate.setUpdate(delete);
            indexUpdates.add(indexUpdate);
        }
    }
    return indexUpdates;
}

Example 7 with ColumnReference

use of org.apache.phoenix.hbase.index.covered.update.ColumnReference in project phoenix by apache.

the class PhoenixTransactionalIndexer method getIndexUpdates.

private Collection<Pair<Mutation, byte[]>> getIndexUpdates(RegionCoprocessorEnvironment env, PhoenixIndexMetaData indexMetaData, Iterator<Mutation> mutationIterator, byte[] txRollbackAttribute) throws IOException {
    Transaction tx = indexMetaData.getTransaction();
    if (tx == null) {
        throw new NullPointerException("Expected to find transaction in metadata for " + env.getRegionInfo().getTable().getNameAsString());
    }
    boolean isRollback = txRollbackAttribute != null;
    boolean isImmutable = indexMetaData.isImmutableRows();
    ResultScanner currentScanner = null;
    TransactionAwareHTable txTable = null;
    // Collect up all mutations in batch
    Map<ImmutableBytesPtr, MultiMutation> mutations = new HashMap<ImmutableBytesPtr, MultiMutation>();
    Map<ImmutableBytesPtr, MultiMutation> findPriorValueMutations;
    if (isImmutable && !isRollback) {
        findPriorValueMutations = new HashMap<ImmutableBytesPtr, MultiMutation>();
    } else {
        findPriorValueMutations = mutations;
    }
    while (mutationIterator.hasNext()) {
        Mutation m = mutationIterator.next();
        // add the mutation to the batch set
        ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow());
        if (mutations != findPriorValueMutations && isDeleteMutation(m)) {
            addMutation(findPriorValueMutations, row, m);
        }
        addMutation(mutations, row, m);
    }
    // Collect the set of mutable ColumnReferences so that we can first
    // run a scan to get the current state. We'll need this to delete
    // the existing index rows.
    List<IndexMaintainer> indexMaintainers = indexMetaData.getIndexMaintainers();
    int estimatedSize = indexMaintainers.size() * 10;
    Set<ColumnReference> mutableColumns = Sets.newHashSetWithExpectedSize(estimatedSize);
    for (IndexMaintainer indexMaintainer : indexMaintainers) {
        // For transactional tables, we use an index maintainer
        // to aid in rollback if there's a KeyValue column in the index. The alternative would be
        // to hold on to all uncommitted index row keys (even ones already sent to HBase) on the
        // client side.
        Set<ColumnReference> allColumns = indexMaintainer.getAllColumns();
        mutableColumns.addAll(allColumns);
    }
    Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>(mutations.size() * 2 * indexMaintainers.size());
    try {
        // this logic will work there too.
        if (!findPriorValueMutations.isEmpty()) {
            List<KeyRange> keys = Lists.newArrayListWithExpectedSize(mutations.size());
            for (ImmutableBytesPtr ptr : findPriorValueMutations.keySet()) {
                keys.add(PVarbinary.INSTANCE.getKeyRange(ptr.copyBytesIfNecessary()));
            }
            Scan scan = new Scan();
            // Project all mutable columns
            for (ColumnReference ref : mutableColumns) {
                scan.addColumn(ref.getFamily(), ref.getQualifier());
            }
            /*
                 * Indexes inherit the storage scheme of the data table which means all the indexes have the same
                 * storage scheme and empty key value qualifier. Note that this assumption would be broken if we start
                 * supporting new indexes over existing data tables to have a different storage scheme than the data
                 * table.
                 */
            byte[] emptyKeyValueQualifier = indexMaintainers.get(0).getEmptyKeyValueQualifier();
            // Project empty key value column
            scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), emptyKeyValueQualifier);
            ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, KeyRange.EVERYTHING_RANGE, null, true, -1);
            scanRanges.initializeScan(scan);
            TableName tableName = env.getRegion().getRegionInfo().getTable();
            HTableInterface htable = env.getTable(tableName);
            txTable = new TransactionAwareHTable(htable);
            txTable.startTx(tx);
            // For rollback, we need to see all versions, including
            // the last committed version as there may be multiple
            // checkpointed versions.
            SkipScanFilter filter = scanRanges.getSkipScanFilter();
            if (isRollback) {
                filter = new SkipScanFilter(filter, true);
                tx.setVisibility(VisibilityLevel.SNAPSHOT_ALL);
            }
            scan.setFilter(filter);
            currentScanner = txTable.getScanner(scan);
        }
        if (isRollback) {
            processRollback(env, indexMetaData, txRollbackAttribute, currentScanner, tx, mutableColumns, indexUpdates, mutations);
        } else {
            processMutation(env, indexMetaData, txRollbackAttribute, currentScanner, tx, mutableColumns, indexUpdates, mutations, findPriorValueMutations);
        }
    } finally {
        if (txTable != null)
            txTable.close();
    }
    return indexUpdates;
}

Example 8 with ColumnReference

use of org.apache.phoenix.hbase.index.covered.update.ColumnReference in project phoenix by apache.

the class IndexMaintainer method write.

// Only called by code older than our 4.10 release
@Deprecated
@Override
public void write(DataOutput output) throws IOException {
    // Encode nIndexSaltBuckets and isMultiTenant together
    WritableUtils.writeVInt(output, (nIndexSaltBuckets + 1) * (isMultiTenant ? -1 : 1));
    // Encode indexedColumns.size() and whether or not there's a viewIndexId
    WritableUtils.writeVInt(output, (indexedColumns.size() + 1) * (viewIndexId != null ? -1 : 1));
    if (viewIndexId != null) {
        output.write(viewIndexId);
    }
    for (ColumnReference ref : indexedColumns) {
        Bytes.writeByteArray(output, ref.getFamily());
        Bytes.writeByteArray(output, ref.getQualifier());
    }
    //TODO remove indexedColumnTypes in the next major release
    for (int i = 0; i < indexedColumnTypes.size(); i++) {
        PDataType type = indexedColumnTypes.get(i);
        WritableUtils.writeVInt(output, type.ordinal());
    }
    // Encode coveredColumns.size() and whether or not this is a local index
    WritableUtils.writeVInt(output, (coveredColumnsMap.size() + 1) * (isLocalIndex ? -1 : 1));
    for (ColumnReference ref : coveredColumnsMap.keySet()) {
        Bytes.writeByteArray(output, ref.getFamily());
        Bytes.writeByteArray(output, ref.getQualifier());
    }
    // TODO: remove when rowKeyOrderOptimizable hack no longer needed
    WritableUtils.writeVInt(output, indexTableName.length * (rowKeyOrderOptimizable ? 1 : -1));
    output.write(indexTableName, 0, indexTableName.length);
    Bytes.writeByteArray(output, dataEmptyKeyValueCF);
    // TODO in order to maintain b/w compatibility encode emptyKeyValueCFPtr.getLength() as a negative value (so we can distinguish between new and old clients)
    // when indexedColumnTypes is removed, remove this 
    WritableUtils.writeVInt(output, -emptyKeyValueCFPtr.getLength());
    output.write(emptyKeyValueCFPtr.get(), emptyKeyValueCFPtr.getOffset(), emptyKeyValueCFPtr.getLength());
    WritableUtils.writeVInt(output, indexedExpressions.size());
    for (Expression expression : indexedExpressions) {
        WritableUtils.writeVInt(output, ExpressionType.valueOf(expression).ordinal());
        expression.write(output);
    }
    rowKeyMetaData.write(output);
    // Encode indexWALDisabled in nDataCFs
    WritableUtils.writeVInt(output, (nDataCFs + 1) * (indexWALDisabled ? -1 : 1));
    // Encode estimatedIndexRowKeyBytes and immutableRows together.
    WritableUtils.writeVInt(output, estimatedIndexRowKeyBytes * (immutableRows ? -1 : 1));
}

Example 9 with ColumnReference

use of org.apache.phoenix.hbase.index.covered.update.ColumnReference in project phoenix by apache.

the class EndToEndCoveredColumnsIndexBuilderIT method testExpectedResultsInTableStateForBatchPuts.

/**
   * Similar to {@link #testExpectedResultsInTableStateForSinglePut()}, but against batches of puts.
   * Previous implementations managed batches by playing current state against each element in the
   * batch, rather than combining all the per-row updates into a single mutation for the batch. This
   * test ensures that we see the correct expected state.
   * @throws Exception on failure
   */
@Test
public void testExpectedResultsInTableStateForBatchPuts() throws Exception {
    long ts = state.ts;
    // build up a list of puts to make, all on the same row
    Put p1 = new Put(row, ts);
    p1.add(family, qual, Bytes.toBytes("v1"));
    Put p2 = new Put(row, ts + 1);
    p2.add(family, qual, Bytes.toBytes("v2"));
    // setup all the verifiers we need. This is just the same as above, but will be called twice
    // since we need to iterate the batch.
    // get all the underlying kvs for the put
    final List<Cell> allKvs = new ArrayList<Cell>(2);
    allKvs.addAll(p2.getFamilyCellMap().get(family));
    allKvs.addAll(p1.getFamilyCellMap().get(family));
    // setup the verifier for the data we expect to write
    // both puts should be put into a single batch
    final ColumnReference familyRef = new ColumnReference(EndToEndCoveredColumnsIndexBuilderIT.family, ColumnReference.ALL_QUALIFIERS);
    VerifyingIndexCodec codec = state.codec;
    // no previous state in the table
    codec.verifiers.add(new ListMatchingVerifier("cleanup state 1", Collections.<Cell>emptyList(), familyRef));
    codec.verifiers.add(new ListMatchingVerifier("put state 1", p1.getFamilyCellMap().get(family), familyRef));
    codec.verifiers.add(new ListMatchingVerifier("cleanup state 2", p1.getFamilyCellMap().get(family), familyRef));
    // kvs from both puts should be in the table now
    codec.verifiers.add(new ListMatchingVerifier("put state 2", allKvs, familyRef));
    // do the actual put (no indexing will actually be done)
    HTable primary = state.table;
    primary.setAutoFlush(false);
    primary.put(Arrays.asList(p1, p2));
    primary.flushCommits();
    // cleanup after ourselves
    cleanup(state);
}

Example 10 with ColumnReference

use of org.apache.phoenix.hbase.index.covered.update.ColumnReference in project phoenix by apache.

the class ScannerBuilder method getColumnFilters.

/**
   * @param columns columns to filter
   * @return filter that will skip any {@link KeyValue} that doesn't match one of the passed columns
   *         and the
   */
private Filter getColumnFilters(Collection<? extends ColumnReference> columns) {
    // each column needs to be added as an OR, so we need to separate them out
    FilterList columnFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE);
    // create a filter that matches each column reference
    for (ColumnReference ref : columns) {
        Filter columnFilter = new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(ref.getFamily()));
        // combine with a match for the qualifier, if the qualifier is a specific qualifier
        if (!Bytes.equals(ColumnReference.ALL_QUALIFIERS, ref.getQualifier())) {
            columnFilter = new FilterList(columnFilter, new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(ref.getQualifier())));
        }
        columnFilters.addFilter(columnFilter);
    }
    return columnFilters;
}