Examples with IBatchID storm.trident.spout.IBatchID used on opensource projects

Example 1 with IBatchID

use of storm.trident.spout.IBatchID in project jstorm by alibaba.

the class TridentBoltExecutor method execute.

@Override
public void execute(Tuple tuple) {
    if (TupleUtils.isTick(tuple)) {
        long now = System.currentTimeMillis();
        if (now - _lastRotate > _messageTimeoutMs) {
            _batches.rotate();
            _lastRotate = now;
        }
        return;
    }
    String batchGroup = _batchGroupIds.get(tuple.getSourceGlobalStreamid());
    if (batchGroup == null) {
        // this is so we can do things like have simple DRPC that doesn't need to use batch processing
        _coordCollector.setCurrBatch(null);
        _bolt.execute(null, tuple);
        _collector.ack(tuple);
        return;
    }
    IBatchID id = (IBatchID) tuple.getValue(0);
    //get transaction id
    //if it already exissts and attempt id is greater than the attempt there
    TrackedBatch tracked = (TrackedBatch) _batches.get(id.getId());
    // failures happen you don't get an explosion in memory usage in the tasks
    if (tracked != null) {
        if (id.getAttemptId() > tracked.attemptId) {
            _batches.remove(id.getId());
            tracked = null;
        } else if (id.getAttemptId() < tracked.attemptId) {
            // no reason to try to execute a previous attempt than we've already seen
            return;
        }
    }
    if (tracked == null) {
        tracked = new TrackedBatch(new BatchInfo(batchGroup, id, _bolt.initBatchState(batchGroup, id)), _coordConditions.get(batchGroup), id.getAttemptId());
        _batches.put(id.getId(), tracked);
    }
    _coordCollector.setCurrBatch(tracked);
    //System.out.println("TRACKED: " + tracked + " " + tuple);
    TupleType t = getTupleType(tuple, tracked);
    if (t == TupleType.COMMIT) {
        tracked.receivedCommit = true;
        checkFinish(tracked, tuple, t);
    } else if (t == TupleType.COORD) {
        int count = tuple.getInteger(1);
        tracked.reportedTasks++;
        tracked.expectedTupleCount += count;
        checkFinish(tracked, tuple, t);
    } else {
        tracked.receivedTuples++;
        boolean success = true;
        try {
            _bolt.execute(tracked.info, tuple);
            if (tracked.condition.expectedTaskReports == 0) {
                success = finishBatch(tracked, tuple);
            }
        } catch (FailedException e) {
            failBatch(tracked, e);
        }
        if (success) {
            _collector.ack(tuple);
        } else {
            _collector.fail(tuple);
        }
    }
    _coordCollector.setCurrBatch(null);
}

Example 2 with IBatchID

use of storm.trident.spout.IBatchID in project storm by nathanmarz.

the class TridentBoltExecutor method execute.

@Override
public void execute(Tuple tuple) {
    if (tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID)) {
        long now = System.currentTimeMillis();
        if (now - _lastRotate > _messageTimeoutMs) {
            _batches.rotate();
            _lastRotate = now;
        }
        return;
    }
    String batchGroup = _batchGroupIds.get(tuple.getSourceGlobalStreamid());
    if (batchGroup == null) {
        // this is so we can do things like have simple DRPC that doesn't need to use batch processing
        _coordCollector.setCurrBatch(null);
        _bolt.execute(null, tuple);
        _collector.ack(tuple);
        return;
    }
    IBatchID id = (IBatchID) tuple.getValue(0);
    //get transaction id
    //if it already exissts and attempt id is greater than the attempt there
    TrackedBatch tracked = (TrackedBatch) _batches.get(id.getId());
    // failures happen you don't get an explosion in memory usage in the tasks
    if (tracked != null) {
        if (id.getAttemptId() > tracked.attemptId) {
            _batches.remove(id.getId());
            tracked = null;
        } else if (id.getAttemptId() < tracked.attemptId) {
            // no reason to try to execute a previous attempt than we've already seen
            return;
        }
    }
    if (tracked == null) {
        tracked = new TrackedBatch(new BatchInfo(batchGroup, id, _bolt.initBatchState(batchGroup, id)), _coordConditions.get(batchGroup), id.getAttemptId());
        _batches.put(id.getId(), tracked);
    }
    _coordCollector.setCurrBatch(tracked);
    //System.out.println("TRACKED: " + tracked + " " + tuple);
    TupleType t = getTupleType(tuple, tracked);
    if (t == TupleType.COMMIT) {
        tracked.receivedCommit = true;
        checkFinish(tracked, tuple, t);
    } else if (t == TupleType.COORD) {
        int count = tuple.getInteger(1);
        tracked.reportedTasks++;
        tracked.expectedTupleCount += count;
        checkFinish(tracked, tuple, t);
    } else {
        tracked.receivedTuples++;
        boolean success = true;
        try {
            _bolt.execute(tracked.info, tuple);
            if (tracked.condition.expectedTaskReports == 0) {
                success = finishBatch(tracked, tuple);
            }
        } catch (FailedException e) {
            failBatch(tracked, e);
        }
        if (success) {
            _collector.ack(tuple);
        } else {
            _collector.fail(tuple);
        }
    }
    _coordCollector.setCurrBatch(null);
}