Examples with BatchGroupId com.alibaba.jstorm.transactional.BatchGroupId used on opensource projects

Example 11 with BatchGroupId

use of com.alibaba.jstorm.transactional.BatchGroupId in project jstorm by alibaba.

the class TransactionBolt method execute.

@Override
public void execute(Tuple input) {
    try {
        String stream = input.getSourceStreamId();
        if (stream.equals(Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID)) {
            processTransactionEvent(input);
        } else {
            List<Object> firstTupleValue = ((Pair<MessageId, List<Object>>) input.getValues().get(0)).getSecond();
            BatchGroupId batchGroupId = (BatchGroupId) firstTupleValue.get(0);
            if (isDiscarded(batchGroupId)) {
                LOG.debug("Tuple was discarded. {}", input);
                return;
            } else if (batchCache.isPendingBatch(batchGroupId, lastSuccessfulBatch)) {
                if (batchCache.cachePendingBatch(batchGroupId, input, lastSuccessfulBatch)) {
                    return;
                }
            }
            currentBatchTracker = getProcessingBatch(batchGroupId, true);
            outputCollector.setCurrBatchTracker(currentBatchTracker);
            processBatchTuple(input);
        }
    } catch (Exception e) {
        LOG.info("Failed to process input={}", input, e);
        throw new RuntimeException(e);
    }
}

Example 12 with BatchGroupId

use of com.alibaba.jstorm.transactional.BatchGroupId in project jstorm by alibaba.

the class TransactionOutputCollector method emit.

@Override
public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple, ICollectorCallback callback) {
    List<Object> tupleWithId = new ArrayList<Object>();
    tupleWithId.add(new BatchGroupId(currBatchTracker.getBatchGroupId()));
    tupleWithId.addAll(tuple);
    return delegate.emit(streamId, anchors, tupleWithId, callback);
}

Example 13 with BatchGroupId

use of com.alibaba.jstorm.transactional.BatchGroupId in project jstorm by alibaba.

the class TransactionSpoutOutputCollector method emitDirect.

@Override
public void emitDirect(int taskId, String streamId, List<Object> tuple, Object messageId, ICollectorCallback callback) {
    try {
        //waitActive();
        lock.readLock().lock();
        List<Object> tupleWithId = new ArrayList<Object>();
        tupleWithId.add(new BatchGroupId(groupId, currBatchId));
        tupleWithId.addAll(tuple);
        delegate.emitDirect(taskId, streamId, tupleWithId, null, (callback != null) ? callback : new CollectorCallback());
    //currBatchInfo.endPos = messageId;
    } finally {
        lock.readLock().unlock();
    }
}

Example 14 with BatchGroupId

use of com.alibaba.jstorm.transactional.BatchGroupId in project jstorm by alibaba.

the class TransactionSpoutOutputCollector method flushBarrier.

public BatchInfo flushBarrier() {
    BatchInfo ret = null;
    try {
        lock.writeLock().lock();
        ret = new BatchInfo(currBatchInfo);
        // flush pending message in outputCollector
        delegate.flush();
        // Emit and flush barrier message to all downstream tasks
        BatchGroupId batchGroupId = new BatchGroupId(groupId, currBatchId);
        for (Entry<Integer, Integer> entry : msgCount.entrySet()) {
            int taskId = entry.getKey();
            int count = entry.setValue(0);
            BatchSnapshot barrierSnapshot = new BatchSnapshot(batchGroupId, count);
            emitDirectByDelegate(taskId, TransactionCommon.BARRIER_STREAM_ID, new Values(batchGroupId, barrierSnapshot), null, null);
        }
        delegate.flush();
        moveToNextBatch();
    } finally {
        lock.writeLock().unlock();
    }
    return ret;
}

Example 15 with BatchGroupId

use of com.alibaba.jstorm.transactional.BatchGroupId in project jstorm by alibaba.

the class SnapshotStateMaster method finishSnapshotStateCommit.

private void finishSnapshotStateCommit(BatchGroupId batchGroupId, SnapshotState snapshotState) {
    Set<Integer> statefulTasks = new HashSet<Integer>();
    statefulTasks.addAll(snapshotState.getSpoutTasks());
    TopoMasterCtrlEvent resp = null;
    boolean isCommitSuccess = false;
    if (batchGroupId.batchId != TransactionCommon.INIT_BATCH_ID) {
        if (snapshotState.isActive()) {
            resp = new TopoMasterCtrlEvent(EventType.transactionCommit);
            resp.addEventValue(batchGroupId);
            statefulTasks.addAll(snapshotState.getStatefulTasks());
            snapshotState.successBatch(batchGroupId.batchId);
            // Try to persist the topology snapshot state. But if any failure happened, just continue.
            try {
                isCommitSuccess = stateOperator.commitState(topologyName, batchGroupId.groupId, topologySnapshotState);
                if (!isCommitSuccess) {
                    LOG.warn("Failed to commit topology state for batch-{}", batchGroupId);
                }
            } catch (Exception e) {
                LOG.warn("Got exception, when committing topology state for batch-{}, {}", batchGroupId, e);
            }
        }
    } else {
        snapshotState.setActive();
        resp = new TopoMasterCtrlEvent(EventType.transactionStart);
        snapshotState.successBatch(batchGroupId.batchId);
        isCommitSuccess = true;
    }
    if (isCommitSuccess) {
        // Ack the init or commit request from spout and stateful bolt
        LOG.debug("Send ack to spouts-{}, event={}", statefulTasks, resp);
        for (Integer spoutTask : statefulTasks) {
            ((BoltCollector) (outputCollector.getDelegate())).emitDirectCtrl(spoutTask, Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID, null, new Values(resp));
        }
    }
    long nextPendingSuccessBatch = snapshotState.getPendingSuccessBatch();
    if (nextPendingSuccessBatch != -1) {
        LOG.info("Try to commit a pending successful batch-{}", nextPendingSuccessBatch);
        finishSnapshotStateCommit(new BatchGroupId(batchGroupId.groupId, nextPendingSuccessBatch), snapshotState);
    }
}