Examples with ConsumerEntryState co.cask.cdap.data2.transaction.queue.ConsumerEntryState used on opensource projects

Example 1 with ConsumerEntryState

use of co.cask.cdap.data2.transaction.queue.ConsumerEntryState in project cdap by caskdata.

the class AbstractStreamFileConsumer method storeInitState.

/**
   * Determines if need to cache initial entry states.
   *
   * @param row Entry row key
   * @param stateValue Entry state value
   * @param cache The cache to fill it if the row key and state value needs to be cached.
   * @return {@code true} if the entry is stored into cache, {@code false} if the entry is not stored.
   */
private boolean storeInitState(byte[] row, byte[] stateValue, Map<byte[], byte[]> cache) {
    if (stateValue == null) {
        // State value shouldn't be null, as the row is only written with state value.
        return false;
    }
    long offset = Bytes.toLong(row, row.length - Longs.BYTES);
    long stateWritePointer = QueueEntryRow.getStateWritePointer(stateValue);
    // In both cases, no need to cache
    if (!readFilter.acceptOffset(offset) || stateWritePointer >= transaction.getWritePointer()) {
        return false;
    }
    // If state is PROCESSED and committed, need to memorize it so that it can be skipped.
    ConsumerEntryState state = QueueEntryRow.getState(stateValue);
    if (state == ConsumerEntryState.PROCESSED && transaction.isVisible(stateWritePointer)) {
        // No need to store the state value.
        cache.put(row, null);
        return true;
    }
    // For group size > 1 case, if the state is not committed, need to memorize current state value for claim entry.
    if (consumerConfig.getDequeueStrategy() == DequeueStrategy.FIFO && consumerConfig.getGroupSize() > 1) {
        int stateInstanceId = QueueEntryRow.getStateInstanceId(stateValue);
        // record the state value as null so that it'll get skipped in the claim entry logic.
        if (stateInstanceId < consumerConfig.getGroupSize() && stateInstanceId != consumerConfig.getInstanceId()) {
            cache.put(row, null);
        } else {
            // Otherwise memorize the value for checkAndPut operation in claim entry.
            cache.put(row, stateValue);
        }
        return true;
    }
    return false;
}

Example 2 with ConsumerEntryState

use of co.cask.cdap.data2.transaction.queue.ConsumerEntryState in project cdap by caskdata.

the class QueueEntryRow method canConsume.

/**
   * Looks at specific queue entry and determines if consumer with given consumer config and current transaction
   * can consume this entry. The answer can be
   * "yes" ({@link co.cask.cdap.data2.transaction.queue.QueueEntryRow.CanConsume#YES},
   * "no"  ({@link co.cask.cdap.data2.transaction.queue.QueueEntryRow.CanConsume#NO},
   * "no" with a hint that given consumer cannot consume any of the entries prior to this one
   *       ({@link co.cask.cdap.data2.transaction.queue.QueueEntryRow.CanConsume#NO_INCLUDING_ALL_OLDER}.
   * The latter one allows for some optimizations when doing scans of entries to be
   * consumed.
   *
   * @param consumerConfig config of the consumer
   * @param transaction current tx
   * @param enqueueWritePointer write pointer used by enqueue of this entry
   * @param counter counter of this entry
   * @param metaValue value of meta column of this entry
   * @param stateValue value of state column of this entry
   * @return one {@link co.cask.cdap.data2.transaction.queue.QueueEntryRow.CanConsume} as per description above.
   */
public static CanConsume canConsume(ConsumerConfig consumerConfig, Transaction transaction, long enqueueWritePointer, int counter, byte[] metaValue, byte[] stateValue) {
    DequeueStrategy dequeueStrategy = consumerConfig.getDequeueStrategy();
    if (stateValue != null) {
        // If the state is written by the current transaction, ignore it, as it's processing
        long stateWritePointer = QueueEntryRow.getStateWritePointer(stateValue);
        if (stateWritePointer == transaction.getWritePointer()) {
            return CanConsume.NO;
        }
        // If the state was updated by a different consumer instance that is still active, ignore this entry.
        // The assumption is, the corresponding instance is either processing (claimed)
        // or going to process it (due to rollback/restart).
        // This only applies to FIFO, as for hash and rr, repartition needs to happen if group size change.
        int stateInstanceId = QueueEntryRow.getStateInstanceId(stateValue);
        if (dequeueStrategy == DequeueStrategy.FIFO && stateInstanceId < consumerConfig.getGroupSize() && stateInstanceId != consumerConfig.getInstanceId()) {
            return CanConsume.NO;
        }
        // If state is PROCESSED and committed, ignore it:
        ConsumerEntryState state = QueueEntryRow.getState(stateValue);
        if (state == ConsumerEntryState.PROCESSED && transaction.isVisible(stateWritePointer)) {
            // Note: here we ignore the long-running transactions, because we know they don't interact with queues.
            if (enqueueWritePointer < transaction.getFirstShortInProgress()) {
                return CanConsume.NO_INCLUDING_ALL_OLDER;
            }
            return CanConsume.NO;
        }
    }
    // Always try to process (claim) if using FIFO. The resolution will be done by atomically setting state to CLAIMED
    int instanceId = consumerConfig.getInstanceId();
    if (dequeueStrategy == DequeueStrategy.ROUND_ROBIN) {
        instanceId = getRoundRobinConsumerInstance(enqueueWritePointer, counter, consumerConfig.getGroupSize());
    } else if (dequeueStrategy == DequeueStrategy.HASH) {
        try {
            Map<String, Integer> hashKeys = QueueEntry.deserializeHashKeys(metaValue);
            instanceId = getHashConsumerInstance(hashKeys, consumerConfig.getHashKey(), consumerConfig.getGroupSize());
        } catch (IOException e) {
            // SHOULD NEVER happen
            throw new RuntimeException(e);
        }
    }
    return consumerConfig.getInstanceId() == instanceId ? CanConsume.YES : CanConsume.NO;
}

Example 3 with ConsumerEntryState

use of co.cask.cdap.data2.transaction.queue.ConsumerEntryState in project cdap by caskdata.

the class InMemoryQueue method dequeue.

public ImmutablePair<List<Key>, List<byte[]>> dequeue(Transaction tx, ConsumerConfig config, ConsumerState consumerState, int maxBatchSize) {
    List<Key> keys = Lists.newArrayListWithCapacity(maxBatchSize);
    List<byte[]> datas = Lists.newArrayListWithCapacity(maxBatchSize);
    NavigableSet<Key> keysToScan = consumerState.startKey == null ? entries.navigableKeySet() : entries.tailMap(consumerState.startKey).navigableKeySet();
    boolean updateStartKey = true;
    // navigableKeySet is immune to concurrent modification
    for (Key key : keysToScan) {
        if (keys.size() >= maxBatchSize) {
            break;
        }
        if (updateStartKey && key.txId < tx.getFirstShortInProgress()) {
            // See QueueEntryRow#canCommit for reason.
            consumerState.startKey = key;
        }
        if (tx.getReadPointer() < key.txId) {
            // the entry is newer than the current transaction. so are all subsequent entries. bail out.
            break;
        } else if (tx.isInProgress(key.txId)) {
            // the entry is in the exclude list of current transaction. There is a chance that visible entries follow.
            // next time we have to revisit this entry
            updateStartKey = false;
            continue;
        }
        Item item = entries.get(key);
        if (item == null) {
            // entry was deleted (evicted or undone) after we started iterating
            continue;
        }
        // check whether this is processed already
        ConsumerEntryState state = item.getConsumerState(config.getGroupId());
        if (ConsumerEntryState.PROCESSED.equals(state)) {
            // already processed but not yet evicted. move on
            continue;
        }
        if (config.getDequeueStrategy().equals(DequeueStrategy.FIFO)) {
            // for FIFO, attempt to claim the entry and return it
            if (item.claim(config)) {
                keys.add(key);
                datas.add(item.entry.getData());
            }
            // else: someone else claimed it, or it was already processed, move on, but we may have to revisit this.
            updateStartKey = false;
            continue;
        }
        // for hash/round robin, if group size is 1, just take it
        if (config.getGroupSize() == 1) {
            keys.add(key);
            datas.add(item.entry.getData());
            updateStartKey = false;
            continue;
        }
        // hash by entry hash key or entry id
        int hash;
        if (config.getDequeueStrategy().equals(DequeueStrategy.ROUND_ROBIN)) {
            hash = key.hashCode();
        } else {
            Integer hashFoundInEntry = item.entry.getHashKey(config.getHashKey());
            hash = hashFoundInEntry == null ? 0 : hashFoundInEntry;
        }
        // modulo of a negative is negative, make sure we're positive or 0.
        if (Math.abs(hash) % config.getGroupSize() == config.getInstanceId()) {
            keys.add(key);
            datas.add(item.entry.getData());
            updateStartKey = false;
        }
    }
    return keys.isEmpty() ? null : ImmutablePair.of(keys, datas);
}

Example 4 with ConsumerEntryState

use of co.cask.cdap.data2.transaction.queue.ConsumerEntryState in project cdap by caskdata.

the class HBaseQueueDebugger method visitRow.

/**
   * @param tx the transaction
   * @param rowKey the key of the row
   * @param stateValue the value of the state column in the row
   * @param queueRowPrefixLength length of the queueRowPrefix
   */
private void visitRow(QueueStatistics stats, Transaction tx, byte[] rowKey, byte[] stateValue, int queueRowPrefixLength) {
    if (stateValue == null) {
        stats.countUnprocessed(1);
        return;
    }
    ConsumerEntryState state = QueueEntryRow.getState(stateValue);
    if (state == ConsumerEntryState.PROCESSED) {
        long writePointer = QueueEntryRow.getWritePointer(rowKey, queueRowPrefixLength);
        stats.recordMinWritePointer(writePointer);
        if (tx.isVisible(writePointer)) {
            stats.countProcessedAndVisible(1);
        } else {
            stats.countProcessedAndNotVisible(1);
        }
    }
}