Examples with IndexedKeyValue org.apache.phoenix.hbase.index.wal.IndexedKeyValue used on opensource projects

Example 1 with IndexedKeyValue

use of org.apache.phoenix.hbase.index.wal.IndexedKeyValue in project phoenix by apache.

the class Indexer method preBatchMutateWithExceptions.

public void preBatchMutateWithExceptions(ObserverContext<RegionCoprocessorEnvironment> c, MiniBatchOperationInProgress<Mutation> miniBatchOp) throws Throwable {
    // first group all the updates for a single row into a single update to be processed
    Map<ImmutableBytesPtr, MultiMutation> mutationsMap = new HashMap<ImmutableBytesPtr, MultiMutation>();
    Durability defaultDurability = Durability.SYNC_WAL;
    if (c.getEnvironment().getRegion() != null) {
        defaultDurability = c.getEnvironment().getRegion().getTableDesc().getDurability();
        defaultDurability = (defaultDurability == Durability.USE_DEFAULT) ? Durability.SYNC_WAL : defaultDurability;
    }
    /*
       * Exclusively lock all rows so we get a consistent read
       * while determining the index updates
       */
    BatchMutateContext context = new BatchMutateContext();
    setBatchMutateContext(c, context);
    Durability durability = Durability.SKIP_WAL;
    boolean copyMutations = false;
    for (int i = 0; i < miniBatchOp.size(); i++) {
        Mutation m = miniBatchOp.getOperation(i);
        if (this.builder.isAtomicOp(m)) {
            miniBatchOp.setOperationStatus(i, IGNORE);
            continue;
        }
        if (this.builder.isEnabled(m)) {
            context.rowLocks.add(lockManager.lockRow(m.getRow(), rowLockWaitDuration));
            Durability effectiveDurablity = (m.getDurability() == Durability.USE_DEFAULT) ? defaultDurability : m.getDurability();
            if (effectiveDurablity.ordinal() > durability.ordinal()) {
                durability = effectiveDurablity;
            }
            // Track whether or not we need to
            ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow());
            if (mutationsMap.containsKey(row)) {
                copyMutations = true;
            } else {
                mutationsMap.put(row, null);
            }
        }
    }
    // early exit if it turns out we don't have any edits
    if (mutationsMap.isEmpty()) {
        return;
    }
    // If we're copying the mutations
    Collection<Mutation> originalMutations;
    Collection<? extends Mutation> mutations;
    if (copyMutations) {
        originalMutations = null;
        mutations = mutationsMap.values();
    } else {
        originalMutations = Lists.newArrayListWithExpectedSize(mutationsMap.size());
        mutations = originalMutations;
    }
    Mutation firstMutation = miniBatchOp.getOperation(0);
    ReplayWrite replayWrite = this.builder.getReplayWrite(firstMutation);
    boolean resetTimeStamp = replayWrite == null;
    long now = EnvironmentEdgeManager.currentTimeMillis();
    byte[] byteNow = Bytes.toBytes(now);
    for (int i = 0; i < miniBatchOp.size(); i++) {
        Mutation m = miniBatchOp.getOperation(i);
        // way optimization go though.
        if (miniBatchOp.getOperationStatus(i) != IGNORE && this.builder.isEnabled(m)) {
            if (resetTimeStamp) {
                // inconsistencies as this case isn't handled correctly currently).
                for (List<Cell> family : m.getFamilyCellMap().values()) {
                    List<KeyValue> familyKVs = KeyValueUtil.ensureKeyValues(family);
                    for (KeyValue kv : familyKVs) {
                        setTimeStamp(kv, byteNow);
                    }
                }
            }
            // the index as they're already written and just being replayed.
            if (replayWrite == ReplayWrite.INDEX_ONLY) {
                miniBatchOp.setOperationStatus(i, NOWRITE);
            }
            // Put and a Delete mutation for the same row).
            if (copyMutations) {
                // Add the mutation to the batch set
                ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow());
                MultiMutation stored = mutationsMap.get(row);
                // we haven't seen this row before, so add it
                if (stored == null) {
                    stored = new MultiMutation(row);
                    mutationsMap.put(row, stored);
                }
                stored.addAll(m);
            } else {
                originalMutations.add(m);
            }
        }
    }
    // dump all the index updates into a single WAL. They will get combined in the end anyways, so
    // don't worry which one we get
    WALEdit edit = miniBatchOp.getWalEdit(0);
    if (edit == null) {
        edit = new WALEdit();
        miniBatchOp.setWalEdit(0, edit);
    }
    if (copyMutations || replayWrite != null) {
        mutations = IndexManagementUtil.flattenMutationsByTimestamp(mutations);
    }
    // get the current span, or just use a null-span to avoid a bunch of if statements
    try (TraceScope scope = Trace.startSpan("Starting to build index updates")) {
        Span current = scope.getSpan();
        if (current == null) {
            current = NullSpan.INSTANCE;
        }
        long start = EnvironmentEdgeManager.currentTimeMillis();
        // get the index updates for all elements in this batch
        Collection<Pair<Mutation, byte[]>> indexUpdates = this.builder.getIndexUpdate(miniBatchOp, mutations);
        long duration = EnvironmentEdgeManager.currentTimeMillis() - start;
        if (duration >= slowIndexPrepareThreshold) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(getCallTooSlowMessage("indexPrepare", duration, slowIndexPrepareThreshold));
            }
            metricSource.incrementNumSlowIndexPrepareCalls();
        }
        metricSource.updateIndexPrepareTime(duration);
        current.addTimelineAnnotation("Built index updates, doing preStep");
        TracingUtils.addAnnotation(current, "index update count", indexUpdates.size());
        byte[] tableName = c.getEnvironment().getRegion().getTableDesc().getTableName().getName();
        Iterator<Pair<Mutation, byte[]>> indexUpdatesItr = indexUpdates.iterator();
        List<Mutation> localUpdates = new ArrayList<Mutation>(indexUpdates.size());
        while (indexUpdatesItr.hasNext()) {
            Pair<Mutation, byte[]> next = indexUpdatesItr.next();
            if (Bytes.compareTo(next.getSecond(), tableName) == 0) {
                localUpdates.add(next.getFirst());
                indexUpdatesItr.remove();
            }
        }
        if (!localUpdates.isEmpty()) {
            miniBatchOp.addOperationsFromCP(0, localUpdates.toArray(new Mutation[localUpdates.size()]));
        }
        if (!indexUpdates.isEmpty()) {
            context.indexUpdates = indexUpdates;
            // write index updates to WAL
            if (durability != Durability.SKIP_WAL) {
                // we have all the WAL durability, so we just update the WAL entry and move on
                for (Pair<Mutation, byte[]> entry : indexUpdates) {
                    edit.add(new IndexedKeyValue(entry.getSecond(), entry.getFirst()));
                }
            }
        }
    }
}

Example 2 with IndexedKeyValue

use of org.apache.phoenix.hbase.index.wal.IndexedKeyValue in project phoenix by apache.

the class Indexer method doPostWithExceptions.

private void doPostWithExceptions(WALEdit edit, Mutation m, final Durability durability) throws Exception {
    //short circuit, if we don't need to do any work
    if (durability == Durability.SKIP_WAL || !this.builder.isEnabled(m) || edit == null) {
        // already did the index update in prePut, so we are done
        return;
    }
    // get the current span, or just use a null-span to avoid a bunch of if statements
    try (TraceScope scope = Trace.startSpan("Completing index writes")) {
        Span current = scope.getSpan();
        if (current == null) {
            current = NullSpan.INSTANCE;
        }
        // there is a little bit of excess here- we iterate all the non-indexed kvs for this check first
        // and then do it again later when getting out the index updates. This should be pretty minor
        // though, compared to the rest of the runtime
        IndexedKeyValue ikv = getFirstIndexedKeyValue(edit);
        /*
           * early exit - we have nothing to write, so we don't need to do anything else. NOTE: we don't
           * release the WAL Rolling lock (INDEX_UPDATE_LOCK) since we never take it in doPre if there are
           * no index updates.
           */
        if (ikv == null) {
            return;
        }
        /*
           * only write the update if we haven't already seen this batch. We only want to write the batch
           * once (this hook gets called with the same WALEdit for each Put/Delete in a batch, which can
           * lead to writing all the index updates for each Put/Delete).
           */
        if (!ikv.getBatchFinished()) {
            Collection<Pair<Mutation, byte[]>> indexUpdates = extractIndexUpdate(edit);
            // already specified on each reference
            try {
                current.addTimelineAnnotation("Actually doing index update for first time");
                writer.writeAndKillYourselfOnFailure(indexUpdates, false);
            } finally {
                // With a custom kill policy, we may throw instead of kill the server.
                // Without doing this in a finally block (at least with the mini cluster),
                // the region server never goes down.
                // mark the batch as having been written. In the single-update case, this never gets check
                // again, but in the batch case, we will check it again (see above).
                ikv.markBatchFinished();
            }
        }
    }
}

Example 3 with IndexedKeyValue

use of org.apache.phoenix.hbase.index.wal.IndexedKeyValue in project phoenix by apache.

the class ReadWriteKeyValuesWithCodecIT method getEdits.

/**
   * @return a bunch of {@link WALEdit}s that test a range of serialization possibilities.
   */
private List<WALEdit> getEdits() {
    // Build up a couple of edits
    List<WALEdit> edits = new ArrayList<WALEdit>();
    Put p = new Put(ROW);
    p.add(FAMILY, null, Bytes.toBytes("v1"));
    WALEdit withPut = new WALEdit();
    addMutation(withPut, p, FAMILY);
    edits.add(withPut);
    Delete d = new Delete(ROW);
    d.deleteColumn(FAMILY, null);
    WALEdit withDelete = new WALEdit();
    addMutation(withDelete, d, FAMILY);
    edits.add(withDelete);
    WALEdit withPutsAndDeletes = new WALEdit();
    addMutation(withPutsAndDeletes, d, FAMILY);
    addMutation(withPutsAndDeletes, p, FAMILY);
    edits.add(withPutsAndDeletes);
    WALEdit justIndexUpdates = new WALEdit();
    byte[] table = Bytes.toBytes("targetTable");
    IndexedKeyValue ikv = new IndexedKeyValue(table, p);
    justIndexUpdates.add(ikv);
    edits.add(justIndexUpdates);
    WALEdit mixed = new WALEdit();
    addMutation(mixed, d, FAMILY);
    mixed.add(ikv);
    addMutation(mixed, p, FAMILY);
    edits.add(mixed);
    return edits;
}

Example 4 with IndexedKeyValue

use of org.apache.phoenix.hbase.index.wal.IndexedKeyValue in project phoenix by apache.

the class Indexer method extractIndexUpdate.

/**
 * Extract the index updates from the WAL Edit
 * @param edit to search for index updates
 * @return the mutations to apply to the index tables
 */
private Collection<Pair<Mutation, byte[]>> extractIndexUpdate(WALEdit edit) {
    // Avoid multiple internal array resizings. Initial size of 64, unless we have fewer cells in the edit
    int initialSize = Math.min(edit.size(), 64);
    Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>(initialSize);
    for (Cell kv : edit.getCells()) {
        if (kv instanceof IndexedKeyValue) {
            IndexedKeyValue ikv = (IndexedKeyValue) kv;
            indexUpdates.add(new Pair<Mutation, byte[]>(ikv.getMutation(), ikv.getIndexTable()));
        }
    }
    return indexUpdates;
}

Example 5 with IndexedKeyValue

use of org.apache.phoenix.hbase.index.wal.IndexedKeyValue in project phoenix by apache.

the class ReadWriteKeyValuesWithCodecTest method getEdits.

/**
 * @return a bunch of {@link WALEdit}s that test a range of serialization possibilities.
 */
private List<WALEdit> getEdits() {
    // Build up a couple of edits
    List<WALEdit> edits = new ArrayList<WALEdit>();
    Put p = new Put(ROW);
    p.add(FAMILY, null, Bytes.toBytes("v1"));
    WALEdit withPut = new WALEdit();
    addMutation(withPut, p, FAMILY);
    edits.add(withPut);
    Delete d = new Delete(ROW);
    d.deleteColumn(FAMILY, null);
    WALEdit withDelete = new WALEdit();
    addMutation(withDelete, d, FAMILY);
    edits.add(withDelete);
    WALEdit withPutsAndDeletes = new WALEdit();
    addMutation(withPutsAndDeletes, d, FAMILY);
    addMutation(withPutsAndDeletes, p, FAMILY);
    edits.add(withPutsAndDeletes);
    WALEdit justIndexUpdates = new WALEdit();
    byte[] table = Bytes.toBytes("targetTable");
    IndexedKeyValue ikv = new IndexedKeyValue(table, p);
    justIndexUpdates.add(ikv);
    edits.add(justIndexUpdates);
    WALEdit mixed = new WALEdit();
    addMutation(mixed, d, FAMILY);
    mixed.add(ikv);
    addMutation(mixed, p, FAMILY);
    edits.add(mixed);
    return edits;
}