Examples with Entry com.alibaba.otter.canal.protocol.CanalEntry.Entry used on opensource projects

Example 16 with Entry

use of com.alibaba.otter.canal.protocol.CanalEntry.Entry in project canal by alibaba.

the class AbstractCanalClientTest method printSummary.

private void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }
    String startPosition = null;
    String endPosition = null;
    if (!CollectionUtils.isEmpty(message.getEntries())) {
        startPosition = buildPositionForDump(message.getEntries().get(0));
        endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
    }
    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    logger.info(context_format, new Object[] { batchId, size, memsize, format.format(new Date()), startPosition, endPosition });
}

Example 17 with Entry

use of com.alibaba.otter.canal.protocol.CanalEntry.Entry in project canal by alibaba.

the class CanalServerWithEmbedded method get.

/**
     * 获取数据，可以指定超时时间.
     * 
     * <pre>
     * 几种case:
     * a. 如果timeout为null，则采用tryGet方式，即时获取
     * b. 如果timeout不为null
     *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
     *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
     * 
     * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
     * </pre>
     */
@Override
public Message get(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    checkSubscribe(clientIdentity);
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    synchronized (canalInstance) {
        // 获取到流式数据中的最后一批获取的位置
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
        if (positionRanges != null) {
            throw new CanalServerException(String.format("clientId:%s has last batch:[%s] isn't ack , maybe loss data", clientIdentity.getClientId(), positionRanges));
        }
        Events<Event> events = null;
        Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
        events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
        if (CollectionUtils.isEmpty(events.getEvents())) {
            logger.debug("get successfully, clientId:{} batchSize:{} but result is null", new Object[] { clientIdentity.getClientId(), batchSize });
            // 返回空包，避免生成batchId，浪费性能
            return new Message(-1, new ArrayList<Entry>());
        } else {
            // 记录到流式信息
            Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
            List<Entry> entrys = Lists.transform(events.getEvents(), new Function<Event, Entry>() {

                public Entry apply(Event input) {
                    return input.getEntry();
                }
            });
            logger.info("get successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), batchSize, entrys.size(), batchId, events.getPositionRange());
            // 直接提交ack
            ack(clientIdentity, batchId);
            return new Message(batchId, entrys);
        }
    }
}

Example 18 with Entry

use of com.alibaba.otter.canal.protocol.CanalEntry.Entry in project canal by alibaba.

the class SessionHandler method messageReceived.

public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
    logger.info("message receives in session handler...");
    ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
    Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
    ClientIdentity clientIdentity = null;
    try {
        switch(packet.getType()) {
            case SUBSCRIPTION:
                Sub sub = Sub.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(sub.getDestination()) && StringUtils.isNotEmpty(sub.getClientId())) {
                    clientIdentity = new ClientIdentity(sub.getDestination(), Short.valueOf(sub.getClientId()), sub.getFilter());
                    MDC.put("destination", clientIdentity.getDestination());
                    // 尝试启动，如果已经启动，忽略
                    if (!embeddedServer.isStart(clientIdentity.getDestination())) {
                        ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
                        if (!runningMonitor.isStart()) {
                            runningMonitor.start();
                        }
                    }
                    embeddedServer.subscribe(clientIdentity);
                    // 设置状态数据
                    ctx.setAttachment(clientIdentity);
                    NettyUtils.ack(ctx.getChannel(), null);
                } else {
                    NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage(), ctx.getChannel(), null);
                }
                break;
            case UNSUBSCRIPTION:
                Unsub unsub = Unsub.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(unsub.getDestination()) && StringUtils.isNotEmpty(unsub.getClientId())) {
                    clientIdentity = new ClientIdentity(unsub.getDestination(), Short.valueOf(unsub.getClientId()), unsub.getFilter());
                    MDC.put("destination", clientIdentity.getDestination());
                    embeddedServer.unsubscribe(clientIdentity);
                    // 尝试关闭
                    stopCanalInstanceIfNecessary(clientIdentity);
                    NettyUtils.ack(ctx.getChannel(), null);
                } else {
                    NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage(), ctx.getChannel(), null);
                }
                break;
            case GET:
                Get get = CanalPacket.Get.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(get.getDestination()) && StringUtils.isNotEmpty(get.getClientId())) {
                    clientIdentity = new ClientIdentity(get.getDestination(), Short.valueOf(get.getClientId()));
                    MDC.put("destination", clientIdentity.getDestination());
                    Message message = null;
                    // } else {
                    if (get.getTimeout() == -1) {
                        // 是否是初始值
                        message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize());
                    } else {
                        TimeUnit unit = convertTimeUnit(get.getUnit());
                        message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize(), get.getTimeout(), unit);
                    }
                    // }
                    Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
                    packetBuilder.setType(PacketType.MESSAGES);
                    Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
                    messageBuilder.setBatchId(message.getId());
                    if (message.getId() != -1 && !CollectionUtils.isEmpty(message.getEntries())) {
                        for (Entry entry : message.getEntries()) {
                            messageBuilder.addMessages(entry.toByteString());
                        }
                    }
                    packetBuilder.setBody(messageBuilder.build().toByteString());
                    // 输出数据
                    NettyUtils.write(ctx.getChannel(), packetBuilder.build().toByteArray(), null);
                } else {
                    NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", get.toString()).getMessage(), ctx.getChannel(), null);
                }
                break;
            case CLIENTACK:
                ClientAck ack = CanalPacket.ClientAck.parseFrom(packet.getBody());
                MDC.put("destination", ack.getDestination());
                if (StringUtils.isNotEmpty(ack.getDestination()) && StringUtils.isNotEmpty(ack.getClientId())) {
                    if (ack.getBatchId() == 0L) {
                        NettyUtils.error(402, MessageFormatter.format("batchId should assign value", ack.toString()).getMessage(), ctx.getChannel(), null);
                    } else if (ack.getBatchId() == -1L) {
                    // -1代表上一次get没有数据，直接忽略之
                    // donothing
                    } else {
                        clientIdentity = new ClientIdentity(ack.getDestination(), Short.valueOf(ack.getClientId()));
                        embeddedServer.ack(clientIdentity, ack.getBatchId());
                    }
                } else {
                    NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage(), ctx.getChannel(), null);
                }
                break;
            case CLIENTROLLBACK:
                ClientRollback rollback = CanalPacket.ClientRollback.parseFrom(packet.getBody());
                MDC.put("destination", rollback.getDestination());
                if (StringUtils.isNotEmpty(rollback.getDestination()) && StringUtils.isNotEmpty(rollback.getClientId())) {
                    clientIdentity = new ClientIdentity(rollback.getDestination(), Short.valueOf(rollback.getClientId()));
                    if (rollback.getBatchId() == 0L) {
                        // 回滚所有批次
                        embeddedServer.rollback(clientIdentity);
                    } else {
                        // 只回滚单个批次
                        embeddedServer.rollback(clientIdentity, rollback.getBatchId());
                    }
                } else {
                    NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage(), ctx.getChannel(), null);
                }
                break;
            default:
                NettyUtils.error(400, MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage(), ctx.getChannel(), null);
                break;
        }
    } catch (Throwable exception) {
        NettyUtils.error(400, MessageFormatter.format("something goes wrong with channel:{}, exception={}", ctx.getChannel(), ExceptionUtils.getStackTrace(exception)).getMessage(), ctx.getChannel(), null);
    } finally {
        MDC.remove("destination");
    }
}

Example 19 with Entry

use of com.alibaba.otter.canal.protocol.CanalEntry.Entry in project canal by alibaba.

the class MysqlDumpTest method testSimple.

@Test
public void testSimple() {
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 4L);
    controller.setConnectionCharset(Charset.forName("UTF-8"));
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
    controller.setMasterPosition(startPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND || entry.getEntryType() == EntryType.HEARTBEAT) {
                    continue;
                }
                RowChange rowChage = null;
                try {
                    rowChage = RowChange.parseFrom(entry.getStoreValue());
                } catch (Exception e) {
                    throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                }
                EventType eventType = rowChage.getEventType();
                System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
                    System.out.println(" sql ----> " + rowChage.getSql());
                }
                for (RowData rowData : rowChage.getRowDatasList()) {
                    if (eventType == EventType.DELETE) {
                        print(rowData.getBeforeColumnsList());
                    } else if (eventType == EventType.INSERT) {
                        print(rowData.getAfterColumnsList());
                    } else {
                        System.out.println("-------> before");
                        print(rowData.getBeforeColumnsList());
                        System.out.println("-------> after");
                        print(rowData.getAfterColumnsList());
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractCanalLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 20 with Entry

use of com.alibaba.otter.canal.protocol.CanalEntry.Entry in project canal by alibaba.

the class MysqlEventParserTest method test_no_position.

@Test
public void test_no_position() throws InterruptedException {
    // 在某个文件下，找不到对应的timestamp数据，会使用106L
    // position进行数据抓取
    final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 60 * 1000);
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", null, new Date().getTime() + 1000 * 1000L);
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(buildAuthentication());
    controller.setMasterPosition(defaultPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        @Override
        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() != EntryType.HEARTBEAT) {
                    entryCount.incrementAndGet();
                    // String logfilename =
                    // entry.getHeader().getLogfileName();
                    // long logfileoffset =
                    // entry.getHeader().getLogfileOffset();
                    long executeTime = entry.getHeader().getExecuteTime();
                    // entryPosition.setJournalName(logfilename);
                    // entryPosition.setPosition(logfileoffset);
                    entryPosition.setTimestamp(executeTime);
                    break;
                }
            }
            if (entryCount.get() > 0) {
                controller.stop();
                timeoutChecker.stop();
                timeoutChecker.touch();
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractCanalLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    // Assert.assertEquals(logfilename, "mysql-bin.000001");
    // Assert.assertEquals(106L, logfileoffset);
    Assert.assertTrue(entryPosition.getTimestamp() < defaultPosition.getTimestamp());
}