Examples with TaskMessage org.apache.storm.messaging.TaskMessage used on opensource projects

Example 1 with TaskMessage

use of org.apache.storm.messaging.TaskMessage in project storm by apache.

the class TransferDrainer method getBundleIterator.

private Iterator<TaskMessage> getBundleIterator(final ArrayList<ArrayList<TaskMessage>> bundle) {
    if (null == bundle) {
        return null;
    }
    return new Iterator<TaskMessage>() {

        private int offset = 0;

        private int size = 0;

        {
            for (ArrayList<TaskMessage> list : bundle) {
                size += list.size();
            }
        }

        private int bundleOffset = 0;

        private Iterator<TaskMessage> iter = bundle.get(bundleOffset).iterator();

        @Override
        public boolean hasNext() {
            return offset < size;
        }

        @Override
        public TaskMessage next() {
            TaskMessage msg;
            if (iter.hasNext()) {
                msg = iter.next();
            } else {
                bundleOffset++;
                iter = bundle.get(bundleOffset).iterator();
                msg = iter.next();
            }
            if (null != msg) {
                offset++;
            }
            return msg;
        }

        @Override
        public void remove() {
            throw new RuntimeException("not supported");
        }
    };
}

Example 2 with TaskMessage

use of org.apache.storm.messaging.TaskMessage in project storm by apache.

the class MessageBatch method buffer.

/**
     * create a buffer containing the encoding of this batch
     */
ChannelBuffer buffer() throws Exception {
    ChannelBufferOutputStream bout = new ChannelBufferOutputStream(ChannelBuffers.directBuffer(encoded_length));
    for (TaskMessage msg : msgs) {
        writeTaskMessage(bout, msg);
    }
    //add a END_OF_BATCH indicator
    ControlMessage.EOB_MESSAGE.write(bout);
    bout.close();
    return bout.buffer();
}

Example 3 with TaskMessage

use of org.apache.storm.messaging.TaskMessage in project storm by apache.

the class MessageDecoder method decode.

/*
     * Each ControlMessage is encoded as:
     *  code (<0) ... short(2)
     * Each TaskMessage is encoded as:
     *  task (>=0) ... short(2)
     *  len ... int(4)
     *  payload ... byte[]     *  
     */
protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buf) throws Exception {
    // Make sure that we have received at least a short 
    long available = buf.readableBytes();
    if (available < 2) {
        //need more data
        return null;
    }
    List<Object> ret = new ArrayList<>();
    // Use while loop, try to decode as more messages as possible in single call
    while (available >= 2) {
        // Mark the current buffer position before reading task/len field
        // because the whole frame might not be in the buffer yet.
        // We will reset the buffer position to the marked position if
        // there's not enough bytes in the buffer.
        buf.markReaderIndex();
        // read the short field
        short code = buf.readShort();
        available -= 2;
        // case 1: Control message
        ControlMessage ctrl_msg = ControlMessage.mkMessage(code);
        if (ctrl_msg != null) {
            if (ctrl_msg == ControlMessage.EOB_MESSAGE) {
                continue;
            } else {
                return ctrl_msg;
            }
        }
        //case 2: SaslTokenMessageRequest
        if (code == SaslMessageToken.IDENTIFIER) {
            // Make sure that we have received at least an integer (length) 
            if (buf.readableBytes() < 4) {
                //need more data
                buf.resetReaderIndex();
                return null;
            }
            // Read the length field.
            int length = buf.readInt();
            if (length <= 0) {
                return new SaslMessageToken(null);
            }
            // Make sure if there's enough bytes in the buffer.
            if (buf.readableBytes() < length) {
                // The whole bytes were not received yet - return null.
                buf.resetReaderIndex();
                return null;
            }
            // There's enough bytes in the buffer. Read it.  
            ChannelBuffer payload = buf.readBytes(length);
            // Return a SaslTokenMessageRequest object
            return new SaslMessageToken(payload.array());
        }
        // Make sure that we have received at least an integer (length)
        if (available < 4) {
            // need more data
            buf.resetReaderIndex();
            break;
        }
        // Read the length field.
        int length = buf.readInt();
        available -= 4;
        if (length <= 0) {
            ret.add(new TaskMessage(code, null));
            break;
        }
        // Make sure if there's enough bytes in the buffer.
        if (available < length) {
            // The whole bytes were not received yet - return null.
            buf.resetReaderIndex();
            break;
        }
        available -= length;
        // There's enough bytes in the buffer. Read it.
        ChannelBuffer payload = buf.readBytes(length);
        // Successfully decoded a frame.
        // Return a TaskMessage object
        ret.add(new TaskMessage(code, payload.array()));
    }
    if (ret.size() == 0) {
        return null;
    } else {
        return ret;
    }
}

Example 4 with TaskMessage

use of org.apache.storm.messaging.TaskMessage in project storm by apache.

the class WorkerState method transfer.

public void transfer(KryoTupleSerializer serializer, List<AddressedTuple> tupleBatch) {
    if (trySerializeLocal) {
        assertCanSerialize(serializer, tupleBatch);
    }
    List<AddressedTuple> local = new ArrayList<>();
    Map<Integer, List<TaskMessage>> remoteMap = new HashMap<>();
    for (AddressedTuple addressedTuple : tupleBatch) {
        int destTask = addressedTuple.getDest();
        if (taskIds.contains(destTask)) {
            // Local task
            local.add(addressedTuple);
        } else {
            // Using java objects directly to avoid performance issues in java code
            if (!remoteMap.containsKey(destTask)) {
                remoteMap.put(destTask, new ArrayList<>());
            }
            remoteMap.get(destTask).add(new TaskMessage(destTask, serializer.serialize(addressedTuple.getTuple())));
        }
    }
    if (!local.isEmpty()) {
        transferLocal(local);
    }
    if (!remoteMap.isEmpty()) {
        transferQueue.publish(remoteMap);
    }
}

Example 5 with TaskMessage

use of org.apache.storm.messaging.TaskMessage in project storm by apache.

the class NettyTest method doTestLoad.

private void doTestLoad(Map<String, Object> stormConf) throws Exception {
    LOG.info("2 test load");
    String reqMessage = "0123456789abcdefghijklmnopqrstuvwxyz";
    IContext context = TransportFactory.makeContext(stormConf, null);
    try {
        AtomicReference<TaskMessage> response = new AtomicReference<>();
        try (IConnection server = context.bind(null, 0, mkConnectionCallback(response::set), null);
            IConnection client = context.connect(null, "localhost", server.getPort(), remoteBpStatus)) {
            waitUntilReady(client, server);
            byte[] messageBytes = reqMessage.getBytes(StandardCharsets.UTF_8);
            send(client, taskId, messageBytes);
            /*
                 * This test sends a broadcast to all connected clients from the server, so we need to wait until the server has registered
                 * the client as connected before sending load metrics.
                 *
                 * It's not enough to wait until the client reports that the channel is open, because the server event loop may not have
                 * finished running channelActive for the new channel. If we send metrics too early, the server will broadcast to no one.
                 *
                 * By waiting for the response here, we ensure that the client will be registered at the server before we send load metrics.
                 */
            waitForNotNull(response);
            Map<Integer, Double> taskToLoad = new HashMap<>();
            taskToLoad.put(1, 0.0);
            taskToLoad.put(2, 1.0);
            server.sendLoadMetrics(taskToLoad);
            List<Integer> tasks = new ArrayList<>();
            tasks.add(1);
            tasks.add(2);
            Testing.whileTimeout(Testing.TEST_TIMEOUT_MS, () -> client.getLoad(tasks).isEmpty(), sleep());
            Map<Integer, Load> load = client.getLoad(tasks);
            assertThat(load.get(1).getBoltLoad(), is(0.0));
            assertThat(load.get(2).getBoltLoad(), is(1.0));
        }
    } finally {
        context.term();
    }
}