Examples with Packet com.alibaba.otter.canal.protocol.CanalPacket.Packet used on opensource projects

Example 1 with Packet

use of com.alibaba.otter.canal.protocol.CanalPacket.Packet in project canal by alibaba.

the class CanalServerWithNettyTest method testAuth.

@Test
public void testAuth() {
    try {
        SocketChannel channel = SocketChannel.open();
        channel.connect(new InetSocketAddress("127.0.0.1", 1088));
        Packet p = Packet.parseFrom(readNextPacket(channel));
        if (p.getVersion() != 1) {
            throw new Exception("unsupported version at this client.");
        }
        if (p.getType() != PacketType.HANDSHAKE) {
            throw new Exception("expect handshake but found other type.");
        }
        //
        Handshake handshake = Handshake.parseFrom(p.getBody());
        System.out.println(handshake.getSupportedCompressionsList());
        //
        ClientAuth ca = ClientAuth.newBuilder().setUsername("").setNetReadTimeout(10000).setNetWriteTimeout(10000).build();
        writeWithHeader(channel, Packet.newBuilder().setType(PacketType.CLIENTAUTHENTICATION).setBody(ca.toByteString()).build().toByteArray());
        //
        p = Packet.parseFrom(readNextPacket(channel));
        if (p.getType() != PacketType.ACK) {
            throw new Exception("unexpected packet type when ack is expected");
        }
        Ack ack = Ack.parseFrom(p.getBody());
        if (ack.getErrorCode() > 0) {
            throw new Exception("something goes wrong when doing authentication: " + ack.getErrorMessage());
        }
        writeWithHeader(channel, Packet.newBuilder().setType(PacketType.SUBSCRIPTION).setBody(Sub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString()).build().toByteArray());
        //
        p = Packet.parseFrom(readNextPacket(channel));
        ack = Ack.parseFrom(p.getBody());
        if (ack.getErrorCode() > 0) {
            throw new Exception("failed to subscribe with reason: " + ack.getErrorMessage());
        }
        for (int i = 0; i < 10; i++) {
            writeWithHeader(channel, Packet.newBuilder().setType(PacketType.GET).setBody(Get.newBuilder().setDestination(DESTINATION).setClientId("1").setFetchSize(10).build().toByteString()).build().toByteArray());
            p = Packet.parseFrom(readNextPacket(channel));
            long batchId = -1L;
            switch(p.getType()) {
                case MESSAGES:
                    {
                        Messages messages = Messages.parseFrom(p.getBody());
                        batchId = messages.getBatchId();
                        break;
                    }
                case ACK:
                    {
                        ack = Ack.parseFrom(p.getBody());
                        if (ack.getErrorCode() > 0) {
                            throw new Exception("failed to subscribe with reason: " + ack.getErrorMessage());
                        }
                        break;
                    }
                default:
                    {
                        throw new Exception("unexpected packet type: " + p.getType());
                    }
            }
            System.out.println("!!!!!!!!!!!!!!!!! " + batchId);
            Thread.sleep(1000L);
            writeWithHeader(channel, Packet.newBuilder().setType(PacketType.CLIENTACK).setBody(ClientAck.newBuilder().setDestination(DESTINATION).setClientId("1").setBatchId(batchId).build().toByteString()).build().toByteArray());
        }
        writeWithHeader(channel, Packet.newBuilder().setType(PacketType.CLIENTROLLBACK).setBody(ClientRollback.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString()).build().toByteArray());
        writeWithHeader(channel, Packet.newBuilder().setType(PacketType.UNSUBSCRIPTION).setBody(Unsub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString()).build().toByteArray());
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 2 with Packet

use of com.alibaba.otter.canal.protocol.CanalPacket.Packet in project canal by alibaba.

the class SimpleCanalConnector method unsubscribe.

@Override
public void unsubscribe() throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return;
    }
    try {
        writeWithHeader(Packet.newBuilder().setType(PacketType.UNSUBSCRIPTION).setBody(Unsub.newBuilder().setDestination(clientIdentity.getDestination()).setClientId(String.valueOf(clientIdentity.getClientId())).build().toByteString()).build().toByteArray());
        // 
        Packet p = Packet.parseFrom(readNextPacket());
        Ack ack = Ack.parseFrom(p.getBody());
        if (ack.getErrorCode() > 0) {
            throw new CanalClientException("failed to unSubscribe with reason: " + ack.getErrorMessage());
        }
    } catch (IOException e) {
        throw new CanalClientException(e);
    }
}

Example 3 with Packet

use of com.alibaba.otter.canal.protocol.CanalPacket.Packet in project canal by alibaba.

the class SimpleCanalConnector method subscribe.

@Override
public void subscribe(String filter) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return;
    }
    try {
        writeWithHeader(Packet.newBuilder().setType(PacketType.SUBSCRIPTION).setBody(Sub.newBuilder().setDestination(clientIdentity.getDestination()).setClientId(String.valueOf(clientIdentity.getClientId())).setFilter(filter != null ? filter : "").build().toByteString()).build().toByteArray());
        // 
        Packet p = Packet.parseFrom(readNextPacket());
        Ack ack = Ack.parseFrom(p.getBody());
        if (ack.getErrorCode() > 0) {
            throw new CanalClientException("failed to subscribe with reason: " + ack.getErrorMessage());
        }
        clientIdentity.setFilter(filter);
    } catch (IOException e) {
        throw new CanalClientException(e);
    }
}

Example 4 with Packet

use of com.alibaba.otter.canal.protocol.CanalPacket.Packet in project canal by alibaba.

the class SimpleCanalConnector method doConnect.

private InetSocketAddress doConnect() throws CanalClientException {
    try {
        channel = SocketChannel.open();
        channel.socket().setSoTimeout(soTimeout);
        SocketAddress address = getAddress();
        if (address == null) {
            address = getNextAddress();
        }
        channel.connect(address);
        readableChannel = Channels.newChannel(channel.socket().getInputStream());
        writableChannel = Channels.newChannel(channel.socket().getOutputStream());
        Packet p = Packet.parseFrom(readNextPacket());
        if (p.getVersion() != 1) {
            throw new CanalClientException("unsupported version at this client.");
        }
        if (p.getType() != PacketType.HANDSHAKE) {
            throw new CanalClientException("expect handshake but found other type.");
        }
        // 
        Handshake handshake = Handshake.parseFrom(p.getBody());
        supportedCompressions.add(handshake.getSupportedCompressions());
        // 
        // seed for auth
        ByteString seed = handshake.getSeeds();
        String newPasswd = password;
        if (password != null) {
            // encode passwd
            newPasswd = SecurityUtil.byte2HexStr(SecurityUtil.scramble411(password.getBytes(), seed.toByteArray()));
        }
        ClientAuth ca = ClientAuth.newBuilder().setUsername(username != null ? username : "").setPassword(ByteString.copyFromUtf8(newPasswd != null ? newPasswd : "")).setNetReadTimeout(idleTimeout).setNetWriteTimeout(idleTimeout).build();
        writeWithHeader(Packet.newBuilder().setType(PacketType.CLIENTAUTHENTICATION).setBody(ca.toByteString()).build().toByteArray());
        // 
        Packet ack = Packet.parseFrom(readNextPacket());
        if (ack.getType() != PacketType.ACK) {
            throw new CanalClientException("unexpected packet type when ack is expected");
        }
        Ack ackBody = Ack.parseFrom(ack.getBody());
        if (ackBody.getErrorCode() > 0) {
            throw new CanalClientException("something goes wrong when doing authentication: " + ackBody.getErrorMessage());
        }
        connected = true;
        return new InetSocketAddress(channel.socket().getLocalAddress(), channel.socket().getLocalPort());
    } catch (IOException | NoSuchAlgorithmException e) {
        throw new CanalClientException(e);
    }
}

Example 5 with Packet

use of com.alibaba.otter.canal.protocol.CanalPacket.Packet in project canal by alibaba.

the class SessionHandler method messageReceived.

@SuppressWarnings({ "deprecation" })
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
    logger.info("message receives in session handler...");
    long start = System.nanoTime();
    ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
    Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
    ClientIdentity clientIdentity = null;
    try {
        switch(packet.getType()) {
            case SUBSCRIPTION:
                Sub sub = Sub.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(sub.getDestination()) && StringUtils.isNotEmpty(sub.getClientId())) {
                    clientIdentity = new ClientIdentity(sub.getDestination(), Short.valueOf(sub.getClientId()), sub.getFilter());
                    MDC.put("destination", clientIdentity.getDestination());
                    // 尝试启动，如果已经启动，忽略
                    if (!embeddedServer.isStart(clientIdentity.getDestination())) {
                        ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
                        if (!runningMonitor.isStart()) {
                            runningMonitor.start();
                        }
                    }
                    embeddedServer.subscribe(clientIdentity);
                    // ctx.setAttachment(clientIdentity);// 设置状态数据
                    byte[] ackBytes = NettyUtils.ackPacket();
                    NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(sub.getDestination(), sub, packet.getType(), ackBytes.length, System.nanoTime() - start));
                } else {
                    byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage());
                    NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(sub.getDestination(), sub, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
                }
                break;
            case UNSUBSCRIPTION:
                Unsub unsub = Unsub.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(unsub.getDestination()) && StringUtils.isNotEmpty(unsub.getClientId())) {
                    clientIdentity = new ClientIdentity(unsub.getDestination(), Short.valueOf(unsub.getClientId()), unsub.getFilter());
                    MDC.put("destination", clientIdentity.getDestination());
                    embeddedServer.unsubscribe(clientIdentity);
                    // 尝试关闭
                    stopCanalInstanceIfNecessary(clientIdentity);
                    byte[] ackBytes = NettyUtils.ackPacket();
                    NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(unsub.getDestination(), unsub, packet.getType(), ackBytes.length, System.nanoTime() - start));
                } else {
                    byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage());
                    NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(unsub.getDestination(), unsub, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
                }
                break;
            case GET:
                Get get = CanalPacket.Get.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(get.getDestination()) && StringUtils.isNotEmpty(get.getClientId())) {
                    clientIdentity = new ClientIdentity(get.getDestination(), Short.valueOf(get.getClientId()));
                    MDC.put("destination", clientIdentity.getDestination());
                    Message message = null;
                    // } else {
                    if (get.getTimeout() == -1) {
                        // 是否是初始值
                        message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize());
                    } else {
                        TimeUnit unit = convertTimeUnit(get.getUnit());
                        message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize(), get.getTimeout(), unit);
                    }
                    if (message.getId() != -1 && message.isRaw()) {
                        List<ByteString> rowEntries = message.getRawEntries();
                        // message size
                        int messageSize = 0;
                        messageSize += com.google.protobuf.CodedOutputStream.computeInt64Size(1, message.getId());
                        int dataSize = 0;
                        for (ByteString rowEntry : rowEntries) {
                            dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntry);
                        }
                        messageSize += dataSize;
                        messageSize += 1 * rowEntries.size();
                        // packet size
                        int size = 0;
                        size += com.google.protobuf.CodedOutputStream.computeEnumSize(3, PacketType.MESSAGES.getNumber());
                        size += com.google.protobuf.CodedOutputStream.computeTagSize(5) + com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize) + messageSize;
                        // recyle bytes
                        // ByteBuffer byteBuffer = (ByteBuffer)
                        // ctx.getAttachment();
                        // if (byteBuffer != null && size <=
                        // byteBuffer.capacity()) {
                        // byteBuffer.clear();
                        // } else {
                        // byteBuffer =
                        // ByteBuffer.allocate(size).order(ByteOrder.BIG_ENDIAN);
                        // ctx.setAttachment(byteBuffer);
                        // }
                        // CodedOutputStream output =
                        // CodedOutputStream.newInstance(byteBuffer);
                        byte[] body = new byte[size];
                        CodedOutputStream output = CodedOutputStream.newInstance(body);
                        output.writeEnum(3, PacketType.MESSAGES.getNumber());
                        output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
                        output.writeRawVarint32(messageSize);
                        // message
                        output.writeInt64(1, message.getId());
                        for (ByteString rowEntry : rowEntries) {
                            output.writeBytes(2, rowEntry);
                        }
                        output.checkNoSpaceLeft();
                        NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(), get, packet.getType(), body.length, System.nanoTime() - start, message.getId() == -1));
                    // output.flush();
                    // byteBuffer.flip();
                    // NettyUtils.write(ctx.getChannel(), byteBuffer,
                    // null);
                    } else {
                        Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
                        packetBuilder.setType(PacketType.MESSAGES).setVersion(NettyUtils.VERSION);
                        Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
                        messageBuilder.setBatchId(message.getId());
                        if (message.getId() != -1) {
                            if (message.isRaw() && !CollectionUtils.isEmpty(message.getRawEntries())) {
                                messageBuilder.addAllMessages(message.getRawEntries());
                            } else if (!CollectionUtils.isEmpty(message.getEntries())) {
                                for (Entry entry : message.getEntries()) {
                                    messageBuilder.addMessages(entry.toByteString());
                                }
                            }
                        }
                        byte[] body = packetBuilder.setBody(messageBuilder.build().toByteString()).build().toByteArray();
                        NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(), get, packet.getType(), body.length, System.nanoTime() - start, // 输出数据
                        message.getId() == -1));
                    }
                } else {
                    byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", get.toString()).getMessage());
                    NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(get.getDestination(), get, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
                }
                break;
            case CLIENTACK:
                ClientAck ack = CanalPacket.ClientAck.parseFrom(packet.getBody());
                MDC.put("destination", ack.getDestination());
                if (StringUtils.isNotEmpty(ack.getDestination()) && StringUtils.isNotEmpty(ack.getClientId())) {
                    if (ack.getBatchId() == 0L) {
                        byte[] errorBytes = NettyUtils.errorPacket(402, MessageFormatter.format("batchId should assign value", ack.toString()).getMessage());
                        NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ack.getDestination(), ack, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 402));
                    } else if (ack.getBatchId() == -1L) {
                    // -1代表上一次get没有数据，直接忽略之
                    // donothing
                    } else {
                        clientIdentity = new ClientIdentity(ack.getDestination(), Short.valueOf(ack.getClientId()));
                        embeddedServer.ack(clientIdentity, ack.getBatchId());
                        new ChannelFutureAggregator(ack.getDestination(), ack, packet.getType(), 0, System.nanoTime() - start).operationComplete(null);
                    }
                } else {
                    byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage());
                    NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ack.getDestination(), ack, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
                }
                break;
            case CLIENTROLLBACK:
                ClientRollback rollback = CanalPacket.ClientRollback.parseFrom(packet.getBody());
                MDC.put("destination", rollback.getDestination());
                if (StringUtils.isNotEmpty(rollback.getDestination()) && StringUtils.isNotEmpty(rollback.getClientId())) {
                    clientIdentity = new ClientIdentity(rollback.getDestination(), Short.valueOf(rollback.getClientId()));
                    if (rollback.getBatchId() == 0L) {
                        // 回滚所有批次
                        embeddedServer.rollback(clientIdentity);
                    } else {
                        // 只回滚单个批次
                        embeddedServer.rollback(clientIdentity, rollback.getBatchId());
                    }
                    new ChannelFutureAggregator(rollback.getDestination(), rollback, packet.getType(), 0, System.nanoTime() - start).operationComplete(null);
                } else {
                    byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage());
                    NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(rollback.getDestination(), rollback, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
                }
                break;
            default:
                byte[] errorBytes = NettyUtils.errorPacket(400, MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage());
                NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel().getRemoteAddress().toString(), null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
                break;
        }
    } catch (Throwable exception) {
        byte[] errorBytes = NettyUtils.errorPacket(400, MessageFormatter.format("something goes wrong with channel:{}, exception={}", ctx.getChannel(), ExceptionUtils.getStackTrace(exception)).getMessage());
        NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel().getRemoteAddress().toString(), null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
    } finally {
        MDC.remove("destination");
    }
}