Examples with BatchGroupId com.alibaba.jstorm.transactional.BatchGroupId used on opensource projects

Example 6 with BatchGroupId

use of com.alibaba.jstorm.transactional.BatchGroupId in project jstorm by alibaba.

the class TransactionSpoutOutputCollector method emit.

@Override
public List<Integer> emit(String streamId, List<Object> tuple, Object messageId, ICollectorCallback callback) {
    try {
        //waitActive();
        lock.readLock().lock();
        List<Object> tupleWithId = new ArrayList<Object>();
        tupleWithId.add(new BatchGroupId(groupId, currBatchId));
        tupleWithId.addAll(tuple);
        delegate.emit(streamId, tupleWithId, null, (callback != null) ? callback : new CollectorCallback());
    //currBatchInfo.endPos = messageId;
    } finally {
        lock.readLock().unlock();
    }
    return null;
}

Example 7 with BatchGroupId

use of com.alibaba.jstorm.transactional.BatchGroupId in project jstorm by alibaba.

the class TransactionSpoutOutputCollector method flushInitBarrier.

public void flushInitBarrier() {
    try {
        lock.writeLock().lock();
        // flush pending message in outputCollector
        delegate.flush();
        BatchGroupId batchGroupId = new BatchGroupId(groupId, TransactionCommon.INIT_BATCH_ID);
        BatchSnapshot barrierSnapshot = new BatchSnapshot(batchGroupId, 0);
        for (Entry<Integer, Integer> entry : msgCount.entrySet()) {
            entry.setValue(0);
            emitDirectByDelegate(entry.getKey(), TransactionCommon.BARRIER_STREAM_ID, new Values(batchGroupId, barrierSnapshot), null, null);
        }
        delegate.flush();
    } finally {
        lock.writeLock().unlock();
    }
}

Example 8 with BatchGroupId

use of com.alibaba.jstorm.transactional.BatchGroupId in project jstorm by alibaba.

the class SnapshotStateMaster method process.

public synchronized void process(Tuple tuple) {
    int taskId = tuple.getSourceTask();
    TopoMasterCtrlEvent event = (TopoMasterCtrlEvent) tuple.getValues().get(0);
    BatchGroupId batchGroupId = null;
    if (event.getEventValue() != null && event.getEventValue().size() > 0) {
        batchGroupId = (BatchGroupId) event.getEventValue().get(0);
    }
    SnapshotState snapshotState = null;
    // If batchGroupId == null, it is initState request
    if (batchGroupId != null) {
        snapshotState = topologySnapshotState.get(batchGroupId.groupId);
        if (snapshotState == null) {
            LOG.warn("unexpected event from task-{}, event={}", taskId, event.toString());
            ;
            return;
        }
    }
    LOG.debug("Received control event from task-{}, event={}", taskId, event);
    boolean isFinished = false;
    switch(event.getEventType()) {
        case transactionInitState:
            for (Entry<Integer, SnapshotState> entry : topologySnapshotState.entrySet()) {
                TransactionState state = entry.getValue().getInitState(taskId);
                int groupId = entry.getKey();
                if (state != null) {
                    TopoMasterCtrlEvent initStateResp = new TopoMasterCtrlEvent(EventType.transactionInitState);
                    initStateResp.addEventValue(state);
                    ((BoltCollector) (outputCollector.getDelegate())).emitDirectCtrl(taskId, Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID, null, new Values(initStateResp));
                }
            }
            break;
        case transactionCommit:
            TransactionState commitState = (TransactionState) event.getEventValue().get(1);
            isFinished = snapshotState.commit(batchGroupId.batchId, taskId, commitState);
            break;
        case transactionAck:
            isFinished = snapshotState.ackEndBolt(batchGroupId.batchId, taskId);
            break;
        case transactionRollback:
            // If receiving rollback request, TM will rollback all tasks which are on the same stream as this task
            stopAndRollback(batchGroupId.groupId, snapshotState.rollback());
            break;
        default:
            LOG.warn("unexpected event type from task-{}, event={}", taskId, event.toString());
            break;
    }
    if (isFinished) {
        finishSnapshotStateCommit(batchGroupId, snapshotState);
    }
    LOG.debug("snapshotState: {}", snapshotState);
}

Example 9 with BatchGroupId

use of com.alibaba.jstorm.transactional.BatchGroupId in project jstorm by alibaba.

the class TransactionBolt method processTransactionEvent.

protected void processTransactionEvent(Tuple input) {
    TopoMasterCtrlEvent event = (TopoMasterCtrlEvent) input.getValues().get(0);
    TransactionState state = null;
    switch(event.getEventType()) {
        case transactionInitState:
            state = (TransactionState) event.getEventValue().get(0);
            initState(state);
            boltStatus = State.ACTIVE;
            break;
        case transactionRollback:
            boltStatus = State.ROLLBACK;
            state = (TransactionState) event.getEventValue().get(0);
            rollback(state);
            break;
        case transactionCommit:
            BatchGroupId batchGroupId = (BatchGroupId) event.getEventValue().get(0);
            ackCommit(batchGroupId);
            break;
        default:
            LOG.warn("Received unexpected transaction event, {}" + event.toString());
            break;
    }
}

Example 10 with BatchGroupId

use of com.alibaba.jstorm.transactional.BatchGroupId in project jstorm by alibaba.

the class TransactionBolt method protoExecute.

@Override
public List<byte[]> protoExecute(byte[] data) {
    if (batchCache == null || !batchCache.isExactlyOnceMode()) {
        return null;
    }
    List<byte[]> ret = new ArrayList<byte[]>();
    if (kryoInput == null) {
        // Bolt has not finished initialize
        ret.add(data);
        return ret;
    }
    kryoInput.setBuffer(data);
    // Skip targetTaskId, timeStamp, isBatch
    kryoInput.setPosition(13);
    // Skip sourceTaskId
    int sourceTaskId = kryoInput.readInt(true);
    int streamId = kryoInput.readInt(true);
    // LOG.info("ProtoExecute: receive init msg from task-{} on stream-{}, data={}", sourceTaskId, streamId, JStormUtils.toPrintableString(data)); 
    if (inputStreamIds.contains(streamId)) {
        // Skip length of batch
        kryoInput.readInt(true);
        // Skip type of tuple value
        kryoInput.readInt(true);
        // Skip length of tuple value
        kryoInput.readInt(true);
        // Skip registration id of BatchGroupId class
        kryoInput.readInt(true);
        // Read BatchGroupId
        BatchGroupId batchGroupId = new BatchGroupId(kryoInput.readInt(true), kryoInput.readLong(true));
        //LOG.info("ProtoExecute: receive msg for batchGroupId-{}", batchGroupId);
        if (batchGroupId.batchId == TransactionCommon.INIT_BATCH_ID) {
            ret.add(data);
        } else {
            if (batchCache.isPendingBatch(batchGroupId, lastSuccessfulBatch)) {
                //LOG.info("Cache batch-{}", batchGroupId);
                if (!batchCache.cachePendingBatch(batchGroupId, data, lastSuccessfulBatch)) {
                    ret.add(data);
                }
            } else {
                ret.add(data);
            }
        }
    } else {
        ret.add(data);
    }
    return ret;
}