Examples with MsgInfo com.alibaba.jstorm.task.execute.MsgInfo used on opensource projects

Example 1 with MsgInfo

use of com.alibaba.jstorm.task.execute.MsgInfo in project jstorm by alibaba.

the class SpoutBatchCollector method sendBatch.

public List<Integer> sendBatch(String outStreamId, String outTaskId, List<MsgInfo> batchTobeFlushed) {
    long startTime = emitTotalTimer.getTime();
    try {
        List<Integer> ret = null;
        Map<Object, List<MsgInfo>> outTasks;
        if (outTaskId != null) {
            outTasks = sendTargets.getBatch(Integer.valueOf(outTaskId), outStreamId, batchTobeFlushed);
        } else {
            outTasks = sendTargets.getBatch(outStreamId, batchTobeFlushed);
        }
        if (outTasks == null || outTasks.size() == 0) {
            // don't need send tuple to other task
            return new ArrayList<Integer>();
        }
        Map<Long, MsgInfo> ackBatch = new HashMap<Long, MsgInfo>();
        for (Map.Entry<Object, List<MsgInfo>> entry : outTasks.entrySet()) {
            Object target = entry.getKey();
            List<Integer> tasks = (target instanceof Integer) ? JStormUtils.mk_list((Integer) target) : ((List<Integer>) target);
            List<MsgInfo> batch = entry.getValue();
            for (int i = 0; i < tasks.size(); i++) {
                Integer t = tasks.get(i);
                List<Object> batchValues = new ArrayList<Object>();
                for (MsgInfo msg : batch) {
                    SpoutMsgInfo msgInfo = (SpoutMsgInfo) msg;
                    Pair<MessageId, List<Object>> pair = new Pair<MessageId, List<Object>>(getMessageId(msgInfo, ackBatch), msgInfo.values);
                    batchValues.add(pair);
                }
                TupleImplExt batchTuple = new TupleImplExt(topology_context, batchValues, task_id, outStreamId, null);
                batchTuple.setTargetTaskId(t);
                batchTuple.setBatchTuple(true);
                transfer_fn.transfer(batchTuple);
            }
            for (MsgInfo msg : batch) {
                if (msg.callback != null) {
                    msg.callback.execute(outStreamId, tasks, msg.values);
                }
            }
        }
        if (ackBatch.size() > 0) {
            sendBatch(Acker.ACKER_INIT_STREAM_ID, null, new ArrayList<MsgInfo>(ackBatch.values()));
        }
        return ret;
    } finally {
        emitTotalTimer.updateTime(startTime);
    }
}

Example 2 with MsgInfo

use of com.alibaba.jstorm.task.execute.MsgInfo in project jstorm by alibaba.

the class SpoutBatchCollector method getMessageId.

protected MessageId getMessageId(SpoutMsgInfo msg, Map<Long, MsgInfo> ackBatch) {
    MessageId msgId = null;
    if (msg.rootId != null) {
        Long as = MessageId.generateId(random);
        msgId = MessageId.makeRootId(msg.rootId, as);
        MsgInfo msgInfo = ackBatch.get(msg.rootId);
        List<Object> ackerTuple;
        if (msgInfo == null) {
            TupleInfo info = new TupleInfo();
            info.setStream(msg.streamId);
            info.setValues(msg.values);
            info.setMessageId(msg.messageId);
            info.setTimestamp(System.currentTimeMillis());
            pending.putHead(msg.rootId, info);
            ackerTuple = JStormUtils.mk_list((Object) msg.rootId, JStormUtils.bit_xor_vals(as), task_id);
            msgInfo = new SpoutMsgInfo(Acker.ACKER_INIT_STREAM_ID, ackerTuple, null, null, null, null);
            ackBatch.put(msg.rootId, msgInfo);
        } else {
            ackerTuple = msgInfo.values;
            ackerTuple.set(1, JStormUtils.bit_xor_vals(ackerTuple.get(1), as));
        }
    }
    return msgId;
}

Example 3 with MsgInfo

use of com.alibaba.jstorm.task.execute.MsgInfo in project jstorm by alibaba.

the class MkFieldsGrouper method batchGrouper.

public void batchGrouper(List<MsgInfo> batch, Map<Object, List<MsgInfo>> ret) {
    for (MsgInfo msg : batch) {
        int hashcode = getHashCode(msg.values);
        int target = out_tasks.get(Math.abs(hashcode % this.out_tasks.size()));
        List<MsgInfo> targetBatch = ret.get(target);
        if (targetBatch == null) {
            targetBatch = new ArrayList<MsgInfo>();
            ret.put(target, targetBatch);
        }
        targetBatch.add(msg);
    }
}

Example 4 with MsgInfo

use of com.alibaba.jstorm.task.execute.MsgInfo in project jstorm by alibaba.

the class TaskSendTargets method getBatch.

public Map<Object, List<MsgInfo>> getBatch(String stream, List<MsgInfo> batch) {
    Map<Object, List<MsgInfo>> outTasks = new HashMap<Object, List<MsgInfo>>();
    // get grouper, then get which task should tuple be sent to.
    Map<String, MkGrouper> componentCrouping = streamComponentgrouper.get(stream);
    if (componentCrouping == null) {
        // if the target component's parallelism is 0, don't need send to
        // them
        LOG.debug("Failed to get Grouper of " + stream + " in " + debugIdStr);
        return outTasks;
    }
    for (Entry<String, MkGrouper> ee : componentCrouping.entrySet()) {
        MkGrouper g = ee.getValue();
        if (GrouperType.direct.equals(g.gettype())) {
            throw new IllegalArgumentException("Cannot do regular emit to direct stream");
        }
        outTasks.putAll(g.grouperBatch(batch));
    }
    /*
        if (isDebug(anchors, root_id)) {

            LOG.info(debugIdStr + stream + " to " + out_tasks);
        }*/
    int num_out_tasks = 0;
    for (Entry<Object, List<MsgInfo>> entry : outTasks.entrySet()) {
        if (entry.getKey() instanceof Integer) {
            num_out_tasks += entry.getValue().size();
        } else {
            num_out_tasks += ((List<Integer>) entry.getKey()).size() * entry.getValue().size();
        }
    }
    taskStats.send_tuple(stream, num_out_tasks);
    return outTasks;
}