Examples with Tuple backtype.storm.tuple.Tuple used on opensource projects

Example 16 with Tuple

use of backtype.storm.tuple.Tuple in project jstorm by alibaba.

the class StatefulBoltExecutor method handleCheckpoint.

@Override
protected void handleCheckpoint(Tuple checkpointTuple, Action action, long txid) {
    LOG.debug("handleCheckPoint with tuple {}, action {}, txid {}", checkpointTuple, action, txid);
    if (action == Action.PREPARE) {
        if (boltInitialized) {
            bolt.prePrepare(txid);
            state.prepareCommit(txid);
            preparedTuples.addAll(collector.ackedTuples());
        } else {
            /*
                 * May be the task restarted in the middle and the state needs be initialized.
                 * Fail fast and trigger recovery.
                  */
            LOG.debug("Failing checkpointTuple, PREPARE received when bolt state is not initialized.");
            collector.fail(checkpointTuple);
            return;
        }
    } else if (action == Action.COMMIT) {
        bolt.preCommit(txid);
        state.commit(txid);
        ack(preparedTuples);
    } else if (action == Action.ROLLBACK) {
        bolt.preRollback();
        state.rollback();
        fail(preparedTuples);
        fail(collector.ackedTuples());
    } else if (action == Action.INITSTATE) {
        if (!boltInitialized) {
            bolt.initState((T) state);
            boltInitialized = true;
            LOG.debug("{} pending tuples to process", pendingTuples.size());
            for (Tuple tuple : pendingTuples) {
                doExecute(tuple);
            }
            pendingTuples.clear();
        } else {
            LOG.debug("Bolt state is already initialized, ignoring tuple {}, action {}, txid {}", checkpointTuple, action, txid);
        }
    }
    collector.emit(CheckpointSpout.CHECKPOINT_STREAM_ID, checkpointTuple, new Values(txid, action));
    collector.delegate.ack(checkpointTuple);
}

Example 17 with Tuple

use of backtype.storm.tuple.Tuple in project jstorm by alibaba.

the class TransactionBolt method finishCurrentBatch.

private void finishCurrentBatch() {
    BatchGroupId batchGroupId = currentBatchTracker.bactchGroupId;
    if (batchGroupId.batchId == TransactionCommon.INIT_BATCH_ID) {
        LOG.info("Received all init events");
        cleanupBuffer(batchGroupId.groupId);
        boltStatus = State.ACTIVE;
    } else {
        commit();
        lastSuccessfulBatch.put(batchGroupId.groupId, batchGroupId.batchId);
    }
    if (downstreamTasks.size() == 0) {
        ackBatch(batchGroupId);
    } else {
        outputCollector.flushBarrier();
    }
    removeProcessingBatch(batchGroupId);
    currentBatchTracker = null;
    LOG.debug("finishCurrentBatch, {}", currentBatchStatusInfo());
    List<Tuple> goingtoProcessBatch = batchCache.getNextPendingTuples(lastSuccessfulBatch);
    if (goingtoProcessBatch != null) {
        BatchGroupId nextGroupBatchId = new BatchGroupId(batchGroupId.groupId, batchGroupId.batchId + 1);
        //LOG.info("Get pending batch-{} which is going to be processed. size={}", nextGroupBatchId, goingtoProcessBatch.size());
        processBatch(nextGroupBatchId, goingtoProcessBatch);
    }
}

Example 18 with Tuple

use of backtype.storm.tuple.Tuple in project jstorm by alibaba.

the class BoltBatchCollector method getMessageId.

@Override
protected MessageId getMessageId(Collection<Tuple> anchors) {
    MessageId ret = null;
    if (anchors != null) {
        Map<Long, Long> anchors_to_ids = new HashMap<Long, Long>();
        long now = System.currentTimeMillis();
        if (now - lastRotate > rotateTime) {
            pending_acks.rotate();
            synchronized (pendingTuples) {
                pendingTuples.rotate();
            }
            lastRotate = now;
        }
        for (Tuple a : anchors) {
            // Long edge_id = MessageId.generateId();
            Long edge_id = MessageId.generateId(random);
            synchronized (pending_acks) {
                put_xor(pending_acks, a, edge_id);
            }
            MessageId messageId = a.getMessageId();
            if (messageId != null) {
                for (Long root_id : messageId.getAnchorsToIds().keySet()) {
                    put_xor(anchors_to_ids, root_id, edge_id);
                }
            }
        }
        ret = MessageId.makeId(anchors_to_ids);
    }
    return ret;
}

Example 19 with Tuple

use of backtype.storm.tuple.Tuple in project jstorm by alibaba.

the class BoltBatchCollector method sendBatch.

public List<Integer> sendBatch(String outStreamId, String outTaskId, List<MsgInfo> batchTobeFlushed, boolean isFlush) {
    final long start = emitTimer.getTime();
    try {
        Map<Object, List<MsgInfo>> outTasks = null;
        if (outTaskId != null) {
            outTasks = sendTargets.getBatch(Integer.valueOf(outTaskId), outStreamId, batchTobeFlushed);
        } else {
            outTasks = sendTargets.getBatch(outStreamId, batchTobeFlushed);
        }
        if (outTasks == null || outTasks.size() == 0) {
        } else {
            for (Map.Entry<Object, List<MsgInfo>> entry : outTasks.entrySet()) {
                Object target = entry.getKey();
                List<Integer> tasks = (target instanceof Integer) ? JStormUtils.mk_list((Integer) target) : ((List<Integer>) target);
                List<MsgInfo> batch = entry.getValue();
                for (Integer t : tasks) {
                    List<Object> batchValues = new ArrayList<Object>();
                    for (MsgInfo msg : batch) {
                        BoltMsgInfo msgInfo = (BoltMsgInfo) msg;
                        Pair<MessageId, List<Object>> pair = new Pair<MessageId, List<Object>>(getMessageId(msgInfo.anchors), msgInfo.values);
                        batchValues.add(pair);
                    }
                    TupleImplExt batchTuple = new TupleImplExt(topologyContext, batchValues, task_id, outStreamId, null);
                    batchTuple.setTargetTaskId(t);
                    batchTuple.setBatchTuple(true);
                    taskTransfer.transfer(batchTuple);
                }
                for (MsgInfo msg : batch) {
                    if (msg.callback != null) {
                        msg.callback.execute(outStreamId, tasks, msg.values);
                    }
                }
            }
        }
        for (MsgInfo msg : batchTobeFlushed) {
            Collection<Tuple> anchors = ((BoltMsgInfo) msg).anchors;
            if (anchors != null && anchors.size() > 0) {
                for (Tuple a : anchors) {
                    synchronized (pendingTuples) {
                        Integer pendingCount = pendingTuples.get(a);
                        if (pendingCount != null) {
                            if (--pendingCount <= 0) {
                                pendingTuples.remove(a);
                            } else {
                                pendingTuples.put(a, pendingCount);
                            }
                        }
                    }
                }
            }
        }
        return null;
    } catch (Exception e) {
        LOG.error("bolt emit", e);
    } finally {
        emitTimer.updateTime(start);
    }
    return new ArrayList<Integer>();
}

Example 20 with Tuple

use of backtype.storm.tuple.Tuple in project jstorm by alibaba.

the class BoltCollector method getMessageId.

protected MessageId getMessageId(Collection<Tuple> anchors) {
    MessageId ret = null;
    if (anchors != null && ackerNum > 0) {
        Map<Long, Long> anchors_to_ids = new HashMap<Long, Long>();
        for (Tuple a : anchors) {
            if (a.getMessageId() != null) {
                Long edge_id = MessageId.generateId(random);
                put_xor(pending_acks, a, edge_id);
                MessageId messageId = a.getMessageId();
                if (messageId != null) {
                    for (Long root_id : messageId.getAnchorsToIds().keySet()) {
                        put_xor(anchors_to_ids, root_id, edge_id);
                    }
                }
            }
        }
        ret = MessageId.makeId(anchors_to_ids);
    }
    return ret;
}