Examples with BatchSnapshot com.alibaba.jstorm.transactional.BatchSnapshot used on opensource projects

Example 1 with BatchSnapshot

use of com.alibaba.jstorm.transactional.BatchSnapshot in project jstorm by alibaba.

the class TransactionSpoutOutputCollector method flushInitBarrier.

public void flushInitBarrier() {
    try {
        lock.writeLock().lock();
        // flush pending message in outputCollector
        delegate.flush();
        BatchGroupId batchGroupId = new BatchGroupId(groupId, TransactionCommon.INIT_BATCH_ID);
        BatchSnapshot barrierSnapshot = new BatchSnapshot(batchGroupId, 0);
        for (Entry<Integer, Integer> entry : msgCount.entrySet()) {
            entry.setValue(0);
            emitDirectByDelegate(entry.getKey(), TransactionCommon.BARRIER_STREAM_ID, new Values(batchGroupId, barrierSnapshot), null, null);
        }
        delegate.flush();
    } finally {
        lock.writeLock().unlock();
    }
}

Example 2 with BatchSnapshot

use of com.alibaba.jstorm.transactional.BatchSnapshot in project jstorm by alibaba.

the class TransactionBolt method processBatchTuple.

public void processBatchTuple(Tuple batchEvent) {
    String stream = batchEvent.getSourceStreamId();
    if (stream.equals(TransactionCommon.BARRIER_STREAM_ID)) {
        Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) batchEvent.getValue(0);
        BatchSnapshot snapshot = (BatchSnapshot) val.getSecond().get(1);
        currentBatchTracker.receiveBarrier(batchEvent.getSourceTask());
        currentBatchTracker.expectedTupleCount += snapshot.getTupleCount();
        LOG.debug("Received batch, stream={}, batchGroupId={}, sourceTask={}, values={}", stream, currentBatchTracker.bactchGroupId, batchEvent.getSourceTask(), snapshot);
        LOG.debug("currentBatchTracker={}, processingBatches={}, pendingBatches={}", currentBatchTracker, processingBatches, batchCache);
    } else {
        for (Object value : batchEvent.getValues()) {
            /*List<Object> firstTupleValue = ((Pair<MessageId, List<Object>>) value).getSecond();
                BatchGroupId batchGroupId = (BatchGroupId) firstTupleValue.get(0);
                if (!batchGroupId.equals(currentBatchTracker.bactchGroupId)) {
                    LOG.warn("batchgroupid-{} is not equal to the once of current batch tracker-{}!", batchGroupId, currentBatchTracker.bactchGroupId);
                }*/
            Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value;
            val.getSecond().remove(0);
            TupleImplExt tuple = new TupleImplExt(topologyContext, val.getSecond(), val.getFirst(), ((TupleImplExt) batchEvent));
            boltExecutor.execute(tuple);
        }
        currentBatchTracker.incrementReceivedCount(batchEvent.getValues().size());
    }
    if (currentBatchTracker.checkFinish()) {
        finishCurrentBatch();
    }
}

Example 3 with BatchSnapshot

use of com.alibaba.jstorm.transactional.BatchSnapshot in project jstorm by alibaba.

the class TransactionSpoutOutputCollector method flushBarrier.

public BatchInfo flushBarrier() {
    BatchInfo ret = null;
    try {
        lock.writeLock().lock();
        ret = new BatchInfo(currBatchInfo);
        // flush pending message in outputCollector
        delegate.flush();
        // Emit and flush barrier message to all downstream tasks
        BatchGroupId batchGroupId = new BatchGroupId(groupId, currBatchId);
        for (Entry<Integer, Integer> entry : msgCount.entrySet()) {
            int taskId = entry.getKey();
            int count = entry.setValue(0);
            BatchSnapshot barrierSnapshot = new BatchSnapshot(batchGroupId, count);
            emitDirectByDelegate(taskId, TransactionCommon.BARRIER_STREAM_ID, new Values(batchGroupId, barrierSnapshot), null, null);
        }
        delegate.flush();
        moveToNextBatch();
    } finally {
        lock.writeLock().unlock();
    }
    return ret;
}

Example 4 with BatchSnapshot

use of com.alibaba.jstorm.transactional.BatchSnapshot in project jstorm by alibaba.

the class TransactionOutputCollector method flushBarrier.

public void flushBarrier() {
    // flush pending message in outputCollector
    flush();
    // Emit and flush barrier message to all downstream tasks
    for (Entry<Integer, CountValue> entry : currBatchTracker.sendMsgCount.entrySet()) {
        int taskId = entry.getKey();
        CountValue count = entry.getValue();
        BatchSnapshot snapshot = new BatchSnapshot(currBatchTracker.getBatchGroupId(), count.count);
        emitDirectByDelegate(taskId, TransactionCommon.BARRIER_STREAM_ID, null, new Values(new BatchGroupId(currBatchTracker.getBatchGroupId()), snapshot), null);
        count.count = 0;
    }
    flush();
}