Examples with Pair com.alibaba.jstorm.utils.Pair used on opensource projects

Example 6 with Pair

use of com.alibaba.jstorm.utils.Pair in project jstorm by alibaba.

the class SpoutBatchCollector method sendBatch.

public List<Integer> sendBatch(String outStreamId, String outTaskId, List<MsgInfo> batchTobeFlushed) {
    long startTime = emitTotalTimer.getTime();
    try {
        List<Integer> ret = null;
        Map<Object, List<MsgInfo>> outTasks;
        if (outTaskId != null) {
            outTasks = sendTargets.getBatch(Integer.valueOf(outTaskId), outStreamId, batchTobeFlushed);
        } else {
            outTasks = sendTargets.getBatch(outStreamId, batchTobeFlushed);
        }
        if (outTasks == null || outTasks.size() == 0) {
            // don't need send tuple to other task
            return new ArrayList<Integer>();
        }
        Map<Long, MsgInfo> ackBatch = new HashMap<Long, MsgInfo>();
        for (Map.Entry<Object, List<MsgInfo>> entry : outTasks.entrySet()) {
            Object target = entry.getKey();
            List<Integer> tasks = (target instanceof Integer) ? JStormUtils.mk_list((Integer) target) : ((List<Integer>) target);
            List<MsgInfo> batch = entry.getValue();
            for (int i = 0; i < tasks.size(); i++) {
                Integer t = tasks.get(i);
                List<Object> batchValues = new ArrayList<Object>();
                for (MsgInfo msg : batch) {
                    SpoutMsgInfo msgInfo = (SpoutMsgInfo) msg;
                    Pair<MessageId, List<Object>> pair = new Pair<MessageId, List<Object>>(getMessageId(msgInfo, ackBatch), msgInfo.values);
                    batchValues.add(pair);
                }
                TupleImplExt batchTuple = new TupleImplExt(topology_context, batchValues, task_id, outStreamId, null);
                batchTuple.setTargetTaskId(t);
                batchTuple.setBatchTuple(true);
                transfer_fn.transfer(batchTuple);
            }
            for (MsgInfo msg : batch) {
                if (msg.callback != null) {
                    msg.callback.execute(outStreamId, tasks, msg.values);
                }
            }
        }
        if (ackBatch.size() > 0) {
            sendBatch(Acker.ACKER_INIT_STREAM_ID, null, new ArrayList<MsgInfo>(ackBatch.values()));
        }
        return ret;
    } finally {
        emitTotalTimer.updateTime(startTime);
    }
}

Example 7 with Pair

use of com.alibaba.jstorm.utils.Pair in project jstorm by alibaba.

the class TransactionBolt method execute.

@Override
public void execute(Tuple input) {
    try {
        String stream = input.getSourceStreamId();
        if (stream.equals(Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID)) {
            processTransactionEvent(input);
        } else {
            List<Object> firstTupleValue = ((Pair<MessageId, List<Object>>) input.getValues().get(0)).getSecond();
            BatchGroupId batchGroupId = (BatchGroupId) firstTupleValue.get(0);
            if (isDiscarded(batchGroupId)) {
                LOG.debug("Tuple was discarded. {}", input);
                return;
            } else if (batchCache.isPendingBatch(batchGroupId, lastSuccessfulBatch)) {
                if (batchCache.cachePendingBatch(batchGroupId, input, lastSuccessfulBatch)) {
                    return;
                }
            }
            currentBatchTracker = getProcessingBatch(batchGroupId, true);
            outputCollector.setCurrBatchTracker(currentBatchTracker);
            processBatchTuple(input);
        }
    } catch (Exception e) {
        LOG.info("Failed to process input={}", input, e);
        throw new RuntimeException(e);
    }
}

Example 8 with Pair

use of com.alibaba.jstorm.utils.Pair in project jstorm by alibaba.

the class TransactionBolt method processBatchTuple.

public void processBatchTuple(Tuple batchEvent) {
    String stream = batchEvent.getSourceStreamId();
    if (stream.equals(TransactionCommon.BARRIER_STREAM_ID)) {
        Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) batchEvent.getValue(0);
        BatchSnapshot snapshot = (BatchSnapshot) val.getSecond().get(1);
        currentBatchTracker.receiveBarrier(batchEvent.getSourceTask());
        currentBatchTracker.expectedTupleCount += snapshot.getTupleCount();
        LOG.debug("Received batch, stream={}, batchGroupId={}, sourceTask={}, values={}", stream, currentBatchTracker.bactchGroupId, batchEvent.getSourceTask(), snapshot);
        LOG.debug("currentBatchTracker={}, processingBatches={}, pendingBatches={}", currentBatchTracker, processingBatches, batchCache);
    } else {
        for (Object value : batchEvent.getValues()) {
            /*List<Object> firstTupleValue = ((Pair<MessageId, List<Object>>) value).getSecond();
                BatchGroupId batchGroupId = (BatchGroupId) firstTupleValue.get(0);
                if (!batchGroupId.equals(currentBatchTracker.bactchGroupId)) {
                    LOG.warn("batchgroupid-{} is not equal to the once of current batch tracker-{}!", batchGroupId, currentBatchTracker.bactchGroupId);
                }*/
            Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value;
            val.getSecond().remove(0);
            TupleImplExt tuple = new TupleImplExt(topologyContext, val.getSecond(), val.getFirst(), ((TupleImplExt) batchEvent));
            boltExecutor.execute(tuple);
        }
        currentBatchTracker.incrementReceivedCount(batchEvent.getValues().size());
    }
    if (currentBatchTracker.checkFinish()) {
        finishCurrentBatch();
    }
}

Example 9 with Pair

use of com.alibaba.jstorm.utils.Pair in project jstorm by alibaba.

the class SpoutExecutors method onEvent.

/**
     * Handle acker message
     *
     * @see com.lmax.disruptor.EventHandler#onEvent(java.lang.Object, long, boolean)
     */
@Override
public void onEvent(Object event, long sequence, boolean endOfBatch) throws Exception {
    try {
        if (event == null) {
            return;
        }
        Runnable runnable = null;
        if (event instanceof Tuple) {
            if (((TupleExt) event).isBatchTuple()) {
                List<Object> values = ((Tuple) event).getValues();
                for (Object value : values) {
                    Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value;
                    TupleImplExt tuple = new TupleImplExt(sysTopologyCtx, val.getSecond(), val.getFirst(), ((TupleImplExt) event));
                    processControlEvent();
                    runnable = processTupleEvent(tuple);
                    if (runnable != null) {
                        runnable.run();
                        runnable = null;
                    }
                }
            } else {
                runnable = processTupleEvent((Tuple) event);
            }
        } else if (event instanceof TimerTrigger.TimerEvent) {
            processTimerEvent((TimerTrigger.TimerEvent) event);
            return;
        } else if (event instanceof IAckMsg) {
            runnable = (Runnable) event;
        } else if (event instanceof Runnable) {
            runnable = (Runnable) event;
        } else {
            LOG.warn("Receive one unknow event-" + event.toString() + " " + idStr);
            return;
        }
        if (runnable != null)
            runnable.run();
    } catch (Throwable e) {
        if (!taskStatus.isShutdown()) {
            LOG.info("Unknow excpetion ", e);
            report_error.report(e);
        }
    }
}

Example 10 with Pair

use of com.alibaba.jstorm.utils.Pair in project jstorm by alibaba.

the class KryoTupleSerializer method serializeTuple.

/**
     * @@@ in the furture, it will skill serialize 'targetTask' through check some flag
     * @see backtype.storm.serialization.ITupleSerializer#serialize(int, backtype.storm.tuple.Tuple)
     */
private void serializeTuple(Output output, Tuple tuple) {
    try {
        boolean isBatchTuple = false;
        if (tuple instanceof TupleExt) {
            output.writeInt(((TupleExt) tuple).getTargetTaskId());
            output.writeLong(((TupleExt) tuple).getCreationTimeStamp());
            output.writeBoolean(((TupleExt) tuple).isBatchTuple());
            isBatchTuple = ((TupleExt) tuple).isBatchTuple();
        }
        output.writeInt(tuple.getSourceTask(), true);
        output.writeInt(_ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()), true);
        if (isBatchTuple) {
            List<Object> values = tuple.getValues();
            int len = values.size();
            output.writeInt(len, true);
            if (_ackerNum > 0) {
                for (Object value : values) {
                    Pair<MessageId, List<Object>> pairValue = (Pair<MessageId, List<Object>>) value;
                    if (pairValue.getFirst() != null) {
                        pairValue.getFirst().serialize(output);
                    } else {
                        output.writeInt(0, true);
                    }
                    _kryo.serializeInto(pairValue.getSecond(), output);
                }
            } else {
                for (Object value : values) {
                    Pair<MessageId, List<Object>> pairValue = (Pair<MessageId, List<Object>>) value;
                    _kryo.serializeInto(pairValue.getSecond(), output);
                }
            }
        } else {
            MessageId msgId = tuple.getMessageId();
            if (msgId != null) {
                msgId.serialize(output);
            } else {
                output.writeInt(0, true);
            }
            _kryo.serializeInto(tuple.getValues(), output);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}