Examples with Message com.alibaba.otter.canal.protocol.Message used on opensource projects

Example 31 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class RabbitMQCanalConnector method process.

private boolean process(byte[] messageData) {
    if (logger.isDebugEnabled()) {
        logger.debug("Get Message: {}", new String(messageData));
    }
    List messageList = new ArrayList<>();
    if (!flatMessage) {
        Message message = CanalMessageDeserializer.deserializer(messageData);
        messageList.add(message);
    } else {
        FlatMessage flatMessage = JSON.parseObject(messageData, FlatMessage.class);
        messageList.add(flatMessage);
    }
    ConsumerBatchMessage batchMessage;
    if (!flatMessage) {
        batchMessage = new ConsumerBatchMessage<Message>(messageList);
    } else {
        batchMessage = new ConsumerBatchMessage<FlatMessage>(messageList);
    }
    try {
        messageBlockingQueue.put(batchMessage);
    } catch (InterruptedException e) {
        logger.error("Put message to queue error", e);
        throw new RuntimeException(e);
    }
    boolean isCompleted;
    try {
        isCompleted = batchMessage.waitFinish(batchProcessTimeout);
    } catch (InterruptedException e) {
        logger.error("Interrupted when waiting messages to be finished.", e);
        throw new RuntimeException(e);
    }
    boolean isSuccess = batchMessage.isSuccess();
    return isCompleted && isSuccess;
}

Example 32 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class SimpleCanalClientPermanceTest method main.

public static void main(String[] args) {
    String destination = "example";
    String ip = "127.0.0.1";
    int batchSize = 1024;
    int count = 0;
    int sum = 0;
    int perSum = 0;
    long start = System.currentTimeMillis();
    long end = 0;
    final ArrayBlockingQueue<Long> queue = new ArrayBlockingQueue<>(100);
    try {
        final CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111), destination, "canal", "canal");
        Thread ackThread = new Thread(() -> {
            while (true) {
                try {
                    long batchId = queue.take();
                    connector.ack(batchId);
                } catch (InterruptedException e) {
                }
            }
        });
        ackThread.start();
        ((SimpleCanalConnector) connector).setLazyParseEntry(true);
        connector.connect();
        connector.subscribe();
        while (true) {
            Message message = connector.getWithoutAck(batchSize, 100L, TimeUnit.MILLISECONDS);
            long batchId = message.getId();
            int size = message.getRawEntries().size();
            sum += size;
            perSum += size;
            count++;
            queue.add(batchId);
            if (count % 10 == 0) {
                end = System.currentTimeMillis();
                if (end - start != 0) {
                    long tps = (perSum * 1000) / (end - start);
                    System.out.println(" total : " + sum + " , current : " + perSum + " , cost : " + (end - start) + " , tps : " + tps);
                    start = end;
                    perSum = 0;
                }
            }
        }
    } catch (Throwable e) {
        e.printStackTrace();
    }
}

Example 33 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class SessionHandler method messageReceived.

@SuppressWarnings({ "deprecation" })
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
    logger.info("message receives in session handler...");
    long start = System.nanoTime();
    ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
    Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
    ClientIdentity clientIdentity = null;
    try {
        switch(packet.getType()) {
            case SUBSCRIPTION:
                Sub sub = Sub.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(sub.getDestination()) && StringUtils.isNotEmpty(sub.getClientId())) {
                    clientIdentity = new ClientIdentity(sub.getDestination(), Short.valueOf(sub.getClientId()), sub.getFilter());
                    MDC.put("destination", clientIdentity.getDestination());
                    // 尝试启动，如果已经启动，忽略
                    if (!embeddedServer.isStart(clientIdentity.getDestination())) {
                        ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
                        if (!runningMonitor.isStart()) {
                            runningMonitor.start();
                        }
                    }
                    embeddedServer.subscribe(clientIdentity);
                    // ctx.setAttachment(clientIdentity);// 设置状态数据
                    byte[] ackBytes = NettyUtils.ackPacket();
                    NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(sub.getDestination(), sub, packet.getType(), ackBytes.length, System.nanoTime() - start));
                } else {
                    byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage());
                    NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(sub.getDestination(), sub, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
                }
                break;
            case UNSUBSCRIPTION:
                Unsub unsub = Unsub.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(unsub.getDestination()) && StringUtils.isNotEmpty(unsub.getClientId())) {
                    clientIdentity = new ClientIdentity(unsub.getDestination(), Short.valueOf(unsub.getClientId()), unsub.getFilter());
                    MDC.put("destination", clientIdentity.getDestination());
                    embeddedServer.unsubscribe(clientIdentity);
                    // 尝试关闭
                    stopCanalInstanceIfNecessary(clientIdentity);
                    byte[] ackBytes = NettyUtils.ackPacket();
                    NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(unsub.getDestination(), unsub, packet.getType(), ackBytes.length, System.nanoTime() - start));
                } else {
                    byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage());
                    NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(unsub.getDestination(), unsub, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
                }
                break;
            case GET:
                Get get = CanalPacket.Get.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(get.getDestination()) && StringUtils.isNotEmpty(get.getClientId())) {
                    clientIdentity = new ClientIdentity(get.getDestination(), Short.valueOf(get.getClientId()));
                    MDC.put("destination", clientIdentity.getDestination());
                    Message message = null;
                    // } else {
                    if (get.getTimeout() == -1) {
                        // 是否是初始值
                        message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize());
                    } else {
                        TimeUnit unit = convertTimeUnit(get.getUnit());
                        message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize(), get.getTimeout(), unit);
                    }
                    if (message.getId() != -1 && message.isRaw()) {
                        List<ByteString> rowEntries = message.getRawEntries();
                        // message size
                        int messageSize = 0;
                        messageSize += com.google.protobuf.CodedOutputStream.computeInt64Size(1, message.getId());
                        int dataSize = 0;
                        for (ByteString rowEntry : rowEntries) {
                            dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntry);
                        }
                        messageSize += dataSize;
                        messageSize += 1 * rowEntries.size();
                        // packet size
                        int size = 0;
                        size += com.google.protobuf.CodedOutputStream.computeEnumSize(3, PacketType.MESSAGES.getNumber());
                        size += com.google.protobuf.CodedOutputStream.computeTagSize(5) + com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize) + messageSize;
                        // recyle bytes
                        // ByteBuffer byteBuffer = (ByteBuffer)
                        // ctx.getAttachment();
                        // if (byteBuffer != null && size <=
                        // byteBuffer.capacity()) {
                        // byteBuffer.clear();
                        // } else {
                        // byteBuffer =
                        // ByteBuffer.allocate(size).order(ByteOrder.BIG_ENDIAN);
                        // ctx.setAttachment(byteBuffer);
                        // }
                        // CodedOutputStream output =
                        // CodedOutputStream.newInstance(byteBuffer);
                        byte[] body = new byte[size];
                        CodedOutputStream output = CodedOutputStream.newInstance(body);
                        output.writeEnum(3, PacketType.MESSAGES.getNumber());
                        output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
                        output.writeRawVarint32(messageSize);
                        // message
                        output.writeInt64(1, message.getId());
                        for (ByteString rowEntry : rowEntries) {
                            output.writeBytes(2, rowEntry);
                        }
                        output.checkNoSpaceLeft();
                        NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(), get, packet.getType(), body.length, System.nanoTime() - start, message.getId() == -1));
                    // output.flush();
                    // byteBuffer.flip();
                    // NettyUtils.write(ctx.getChannel(), byteBuffer,
                    // null);
                    } else {
                        Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
                        packetBuilder.setType(PacketType.MESSAGES).setVersion(NettyUtils.VERSION);
                        Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
                        messageBuilder.setBatchId(message.getId());
                        if (message.getId() != -1) {
                            if (message.isRaw() && !CollectionUtils.isEmpty(message.getRawEntries())) {
                                messageBuilder.addAllMessages(message.getRawEntries());
                            } else if (!CollectionUtils.isEmpty(message.getEntries())) {
                                for (Entry entry : message.getEntries()) {
                                    messageBuilder.addMessages(entry.toByteString());
                                }
                            }
                        }
                        byte[] body = packetBuilder.setBody(messageBuilder.build().toByteString()).build().toByteArray();
                        NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(), get, packet.getType(), body.length, System.nanoTime() - start, // 输出数据
                        message.getId() == -1));
                    }
                } else {
                    byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", get.toString()).getMessage());
                    NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(get.getDestination(), get, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
                }
                break;
            case CLIENTACK:
                ClientAck ack = CanalPacket.ClientAck.parseFrom(packet.getBody());
                MDC.put("destination", ack.getDestination());
                if (StringUtils.isNotEmpty(ack.getDestination()) && StringUtils.isNotEmpty(ack.getClientId())) {
                    if (ack.getBatchId() == 0L) {
                        byte[] errorBytes = NettyUtils.errorPacket(402, MessageFormatter.format("batchId should assign value", ack.toString()).getMessage());
                        NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ack.getDestination(), ack, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 402));
                    } else if (ack.getBatchId() == -1L) {
                    // -1代表上一次get没有数据，直接忽略之
                    // donothing
                    } else {
                        clientIdentity = new ClientIdentity(ack.getDestination(), Short.valueOf(ack.getClientId()));
                        embeddedServer.ack(clientIdentity, ack.getBatchId());
                        new ChannelFutureAggregator(ack.getDestination(), ack, packet.getType(), 0, System.nanoTime() - start).operationComplete(null);
                    }
                } else {
                    byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage());
                    NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ack.getDestination(), ack, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
                }
                break;
            case CLIENTROLLBACK:
                ClientRollback rollback = CanalPacket.ClientRollback.parseFrom(packet.getBody());
                MDC.put("destination", rollback.getDestination());
                if (StringUtils.isNotEmpty(rollback.getDestination()) && StringUtils.isNotEmpty(rollback.getClientId())) {
                    clientIdentity = new ClientIdentity(rollback.getDestination(), Short.valueOf(rollback.getClientId()));
                    if (rollback.getBatchId() == 0L) {
                        // 回滚所有批次
                        embeddedServer.rollback(clientIdentity);
                    } else {
                        // 只回滚单个批次
                        embeddedServer.rollback(clientIdentity, rollback.getBatchId());
                    }
                    new ChannelFutureAggregator(rollback.getDestination(), rollback, packet.getType(), 0, System.nanoTime() - start).operationComplete(null);
                } else {
                    byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage());
                    NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(rollback.getDestination(), rollback, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
                }
                break;
            default:
                byte[] errorBytes = NettyUtils.errorPacket(400, MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage());
                NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel().getRemoteAddress().toString(), null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
                break;
        }
    } catch (Throwable exception) {
        byte[] errorBytes = NettyUtils.errorPacket(400, MessageFormatter.format("something goes wrong with channel:{}, exception={}", ctx.getChannel(), ExceptionUtils.getStackTrace(exception)).getMessage());
        NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel().getRemoteAddress().toString(), null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
    } finally {
        MDC.remove("destination");
    }
}

Example 34 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class ProtocolTest method testSimple.

@Test(expected = CanalClientException.class)
public void testSimple() throws IOException {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName("mysql-bin.000001");
    headerBuilder.setLogfileOffset(1024);
    headerBuilder.setExecuteTime(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.ROWDATA);
    Entry entry = entryBuilder.build();
    Message message = new Message(3, true, Arrays.asList(entry.toByteString()));
    byte[] body = buildData(message);
    Packet packet = Packet.parseFrom(body);
    switch(packet.getType()) {
        case MESSAGES:
            {
                if (!packet.getCompression().equals(Compression.NONE)) {
                    throw new CanalClientException("compression is not supported in this connector");
                }
                Messages messages = Messages.parseFrom(packet.getBody());
                Message result = new Message(messages.getBatchId());
                for (ByteString byteString : messages.getMessagesList()) {
                    result.addEntry(Entry.parseFrom(byteString));
                }
                System.out.println(result);
                break;
            }
        default:
            {
                throw new CanalClientException("unexpected packet type: " + packet.getType());
            }
    }
}

Example 35 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class BaseCanalServerWithEmbededTest method testSwitch.

// @Test
public void testSwitch() {
    int maxEmptyCount = 10;
    int emptyCount = 0;
    int totalCount = 0;
    int thresold = 50;
    int batchSize = 11;
    server.subscribe(clientIdentity);
    while (emptyCount < maxEmptyCount) {
        Message message = server.get(clientIdentity, batchSize);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }
            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
            if ((totalCount + 1) % 100 >= thresold && (totalCount + 1) % 100 <= thresold + batchSize) {
                CanalEventParser eventParser = server.getCanalInstances().get(DESTINATION).getEventParser();
                if (eventParser instanceof CanalHASwitchable) {
                    // 执行切换
                    ((CanalHASwitchable) eventParser).doSwitch();
                    try {
                        // 等待parser启动
                        Thread.sleep(5 * 1000);
                    } catch (InterruptedException e) {
                        Assert.fail();
                    }
                }
            }
        }
    }
    System.out.println("!!!!!! testGet totalCount : " + totalCount);
    server.unsubscribe(clientIdentity);
}