Examples with MessageId backtype.storm.tuple.MessageId used on opensource projects

Example 1 with MessageId

use of backtype.storm.tuple.MessageId in project jstorm by alibaba.

the class KryoTupleDeserializer method deserialize.

public Tuple deserialize(Input input) {
    int targetTaskId = 0;
    long timeStamp = 0l;
    int taskId = 0;
    int streamId = 0;
    String componentName = null;
    String streamName = null;
    MessageId id = null;
    boolean isBatchTuple = false;
    try {
        targetTaskId = input.readInt();
        timeStamp = input.readLong();
        isBatchTuple = input.readBoolean();
        taskId = input.readInt(true);
        streamId = input.readInt(true);
        componentName = _context.getComponentId(taskId);
        streamName = _ids.getStreamName(componentName, streamId);
        List<Object> values = null;
        if (isBatchTuple) {
            values = new ArrayList<Object>();
            int len = input.readInt(true);
            if (_ackerNum > 0) {
                for (int i = 0; i < len; i++) {
                    values.add(new Pair<MessageId, List<Object>>(MessageId.deserialize(input), _kryo.deserializeFrom(input)));
                }
            } else {
                for (int i = 0; i < len; i++) {
                    values.add(new Pair<MessageId, List<Object>>(null, _kryo.deserializeFrom(input)));
                }
            }
        } else {
            id = MessageId.deserialize(input);
            values = _kryo.deserializeFrom(input);
        }
        TupleImplExt tuple = new TupleImplExt(_context, values, taskId, streamName, id);
        tuple.setBatchTuple(isBatchTuple);
        tuple.setTargetTaskId(targetTaskId);
        tuple.setCreationTimeStamp(timeStamp);
        return tuple;
    } catch (Exception e) {
        StringBuilder sb = new StringBuilder();
        sb.append("Deserialize error:");
        sb.append("targetTaskId:").append(targetTaskId);
        sb.append(",creationTimeStamp:").append(timeStamp);
        sb.append(",isBatchTuple:").append(isBatchTuple);
        sb.append(",taskId:").append(taskId);
        sb.append(",streamId:").append(streamId);
        sb.append(",componentName:").append(componentName);
        sb.append(",streamName:").append(streamName);
        sb.append(",MessageId").append(id);
        LOG.error("Kryo error!!! {} {}", sb.toString(), e);
        throw new RuntimeException(e);
    }
}

Example 2 with MessageId

use of backtype.storm.tuple.MessageId in project jstorm by alibaba.

the class BoltBatchCollector method sendAckTuple.

private boolean sendAckTuple(Tuple input) {
    boolean ret = false;
    Integer pendingCount;
    synchronized (pendingTuples) {
        pendingCount = pendingTuples.get(input);
    }
    if (pendingCount == null || pendingCount <= 0) {
        long ack_val = 0L;
        Object pend_val = pending_acks.remove(input);
        if (pend_val != null) {
            ack_val = (Long) (pend_val);
        }
        MessageId messageId = input.getMessageId();
        if (messageId != null) {
            for (Map.Entry<Long, Long> e : messageId.getAnchorsToIds().entrySet()) {
                List<Object> ackTuple = JStormUtils.mk_list((Object) e.getKey(), JStormUtils.bit_xor(e.getValue(), ack_val));
                sendBoltMsg(Acker.ACKER_ACK_STREAM_ID, null, ackTuple, null, null);
            }
        }
        ret = true;
    }
    return ret;
}

Example 3 with MessageId

use of backtype.storm.tuple.MessageId in project jstorm by alibaba.

the class BoltBatchCollector method getMessageId.

@Override
protected MessageId getMessageId(Collection<Tuple> anchors) {
    MessageId ret = null;
    if (anchors != null) {
        Map<Long, Long> anchors_to_ids = new HashMap<Long, Long>();
        long now = System.currentTimeMillis();
        if (now - lastRotate > rotateTime) {
            pending_acks.rotate();
            synchronized (pendingTuples) {
                pendingTuples.rotate();
            }
            lastRotate = now;
        }
        for (Tuple a : anchors) {
            // Long edge_id = MessageId.generateId();
            Long edge_id = MessageId.generateId(random);
            synchronized (pending_acks) {
                put_xor(pending_acks, a, edge_id);
            }
            MessageId messageId = a.getMessageId();
            if (messageId != null) {
                for (Long root_id : messageId.getAnchorsToIds().keySet()) {
                    put_xor(anchors_to_ids, root_id, edge_id);
                }
            }
        }
        ret = MessageId.makeId(anchors_to_ids);
    }
    return ret;
}

Example 4 with MessageId

use of backtype.storm.tuple.MessageId in project jstorm by alibaba.

the class BoltBatchCollector method sendBatch.

public List<Integer> sendBatch(String outStreamId, String outTaskId, List<MsgInfo> batchTobeFlushed, boolean isFlush) {
    final long start = emitTimer.getTime();
    try {
        Map<Object, List<MsgInfo>> outTasks = null;
        if (outTaskId != null) {
            outTasks = sendTargets.getBatch(Integer.valueOf(outTaskId), outStreamId, batchTobeFlushed);
        } else {
            outTasks = sendTargets.getBatch(outStreamId, batchTobeFlushed);
        }
        if (outTasks == null || outTasks.size() == 0) {
        } else {
            for (Map.Entry<Object, List<MsgInfo>> entry : outTasks.entrySet()) {
                Object target = entry.getKey();
                List<Integer> tasks = (target instanceof Integer) ? JStormUtils.mk_list((Integer) target) : ((List<Integer>) target);
                List<MsgInfo> batch = entry.getValue();
                for (Integer t : tasks) {
                    List<Object> batchValues = new ArrayList<Object>();
                    for (MsgInfo msg : batch) {
                        BoltMsgInfo msgInfo = (BoltMsgInfo) msg;
                        Pair<MessageId, List<Object>> pair = new Pair<MessageId, List<Object>>(getMessageId(msgInfo.anchors), msgInfo.values);
                        batchValues.add(pair);
                    }
                    TupleImplExt batchTuple = new TupleImplExt(topologyContext, batchValues, task_id, outStreamId, null);
                    batchTuple.setTargetTaskId(t);
                    batchTuple.setBatchTuple(true);
                    taskTransfer.transfer(batchTuple);
                }
                for (MsgInfo msg : batch) {
                    if (msg.callback != null) {
                        msg.callback.execute(outStreamId, tasks, msg.values);
                    }
                }
            }
        }
        for (MsgInfo msg : batchTobeFlushed) {
            Collection<Tuple> anchors = ((BoltMsgInfo) msg).anchors;
            if (anchors != null && anchors.size() > 0) {
                for (Tuple a : anchors) {
                    synchronized (pendingTuples) {
                        Integer pendingCount = pendingTuples.get(a);
                        if (pendingCount != null) {
                            if (--pendingCount <= 0) {
                                pendingTuples.remove(a);
                            } else {
                                pendingTuples.put(a, pendingCount);
                            }
                        }
                    }
                }
            }
        }
        return null;
    } catch (Exception e) {
        LOG.error("bolt emit", e);
    } finally {
        emitTimer.updateTime(start);
    }
    return new ArrayList<Integer>();
}

Example 5 with MessageId

use of backtype.storm.tuple.MessageId in project jstorm by alibaba.

the class BoltCollector method getMessageId.

protected MessageId getMessageId(Collection<Tuple> anchors) {
    MessageId ret = null;
    if (anchors != null && ackerNum > 0) {
        Map<Long, Long> anchors_to_ids = new HashMap<Long, Long>();
        for (Tuple a : anchors) {
            if (a.getMessageId() != null) {
                Long edge_id = MessageId.generateId(random);
                put_xor(pending_acks, a, edge_id);
                MessageId messageId = a.getMessageId();
                if (messageId != null) {
                    for (Long root_id : messageId.getAnchorsToIds().keySet()) {
                        put_xor(anchors_to_ids, root_id, edge_id);
                    }
                }
            }
        }
        ret = MessageId.makeId(anchors_to_ids);
    }
    return ret;
}