Examples with Entry com.alibaba.otter.canal.protocol.CanalEntry.Entry used on opensource projects

Example 1 with Entry

use of com.alibaba.otter.canal.protocol.CanalEntry.Entry in project canal by alibaba.

the class AbstractCanalClientTest method printEntry.

protected void printEntry(List<Entry> entrys) {
    for (Entry entry : entrys) {
        long executeTime = entry.getHeader().getExecuteTime();
        long delayTime = new Date().getTime() - executeTime;
        if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
            if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN) {
                TransactionBegin begin = null;
                try {
                    begin = TransactionBegin.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务头信息，执行的线程id，事务耗时
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
                logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
            } else if (entry.getEntryType() == EntryType.TRANSACTIONEND) {
                TransactionEnd end = null;
                try {
                    end = TransactionEnd.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务提交信息，事务id
                logger.info("----------------\n");
                logger.info(" END ----> transaction id: {}", end.getTransactionId());
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
            }
            continue;
        }
        if (entry.getEntryType() == EntryType.ROWDATA) {
            RowChange rowChage = null;
            try {
                rowChage = RowChange.parseFrom(entry.getStoreValue());
            } catch (Exception e) {
                throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
            }
            EventType eventType = rowChage.getEventType();
            logger.info(row_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType, String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
            if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
                logger.info(" sql ----> " + rowChage.getSql() + SEP);
                continue;
            }
            for (RowData rowData : rowChage.getRowDatasList()) {
                if (eventType == EventType.DELETE) {
                    printColumn(rowData.getBeforeColumnsList());
                } else if (eventType == EventType.INSERT) {
                    printColumn(rowData.getAfterColumnsList());
                } else {
                    printColumn(rowData.getAfterColumnsList());
                }
            }
        }
    }
}

Example 2 with Entry

use of com.alibaba.otter.canal.protocol.CanalEntry.Entry in project canal by alibaba.

the class AbstractEventParser method buildHeartBeatTimeTask.

protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
    return new TimerTask() {

        public void run() {
            try {
                if (exception == null || lastEntryTime > 0) {
                    // 如果未出现异常，或者有第一条正常数据
                    long now = System.currentTimeMillis();
                    long inteval = (now - lastEntryTime) / 1000;
                    if (inteval >= detectingIntervalInSeconds) {
                        Header.Builder headerBuilder = Header.newBuilder();
                        headerBuilder.setExecuteTime(now);
                        Entry.Builder entryBuilder = Entry.newBuilder();
                        entryBuilder.setHeader(headerBuilder.build());
                        entryBuilder.setEntryType(EntryType.HEARTBEAT);
                        Entry entry = entryBuilder.build();
                        // 提交到sink中，目前不会提交到store中，会在sink中进行忽略
                        consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
                    }
                }
            } catch (Throwable e) {
                logger.warn("heartBeat run failed " + ExceptionUtils.getStackTrace(e));
            }
        }
    };
}

Example 3 with Entry

use of com.alibaba.otter.canal.protocol.CanalEntry.Entry in project canal by alibaba.

the class AbstractEventParser method start.

public void start() {
    super.start();
    MDC.put("destination", destination);
    // 配置transaction buffer
    // 初始化缓冲队列
    // 设置buffer大小
    transactionBuffer.setBufferSize(transactionSize);
    transactionBuffer.start();
    // 构造bin log parser
    // 初始化一下BinLogParser
    binlogParser = buildParser();
    binlogParser.start();
    // 启动工作线程
    parseThread = new Thread(new Runnable() {

        public void run() {
            MDC.put("destination", String.valueOf(destination));
            ErosaConnection erosaConnection = null;
            while (running) {
                try {
                    // 开始执行replication
                    // 1. 构造Erosa连接
                    erosaConnection = buildErosaConnection();
                    // 2. 启动一个心跳线程
                    startHeartBeat(erosaConnection);
                    // 3. 执行dump前的准备工作
                    preDump(erosaConnection);
                    // 链接
                    erosaConnection.connect();
                    // 4. 获取最后的位置信息
                    EntryPosition position = findStartPosition(erosaConnection);
                    final EntryPosition startPosition = position;
                    if (startPosition == null) {
                        throw new CanalParseException("can't find start position for " + destination);
                    }
                    logger.info("find start position : {}", startPosition.toString());
                    // 重新链接，因为在找position过程中可能有状态，需要断开后重建
                    erosaConnection.reconnect();
                    final SinkFunction sinkHandler = new SinkFunction<EVENT>() {

                        private LogPosition lastPosition;

                        public boolean sink(EVENT event) {
                            try {
                                CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
                                if (!running) {
                                    return false;
                                }
                                if (entry != null) {
                                    // 有正常数据流过，清空exception
                                    exception = null;
                                    transactionBuffer.add(entry);
                                    // 记录一下对应的positions
                                    this.lastPosition = buildLastPosition(entry);
                                    // 记录一下最后一次有数据的时间
                                    lastEntryTime = System.currentTimeMillis();
                                }
                                return running;
                            } catch (TableIdNotFoundException e) {
                                throw e;
                            } catch (Throwable e) {
                                // 记录一下，出错的位点信息
                                processSinkError(e, this.lastPosition, startPosition.getJournalName(), startPosition.getPosition());
                                // 继续抛出异常，让上层统一感知
                                throw new CanalParseException(e);
                            }
                        }
                    };
                    // 4. 开始dump数据
                    if (StringUtils.isEmpty(startPosition.getJournalName()) && startPosition.getTimestamp() != null) {
                        erosaConnection.dump(startPosition.getTimestamp(), sinkHandler);
                    } else {
                        erosaConnection.dump(startPosition.getJournalName(), startPosition.getPosition(), sinkHandler);
                    }
                } catch (TableIdNotFoundException e) {
                    exception = e;
                    // 特殊处理TableIdNotFound异常,出现这样的异常，一种可能就是起始的position是一个事务当中，导致tablemap
                    // Event时间没解析过
                    needTransactionPosition.compareAndSet(false, true);
                    logger.error(String.format("dump address %s has an error, retrying. caused by ", runningInfo.getAddress().toString()), e);
                } catch (Throwable e) {
                    processDumpError(e);
                    exception = e;
                    if (!running) {
                        if (!(e instanceof java.nio.channels.ClosedByInterruptException || e.getCause() instanceof java.nio.channels.ClosedByInterruptException)) {
                            throw new CanalParseException(String.format("dump address %s has an error, retrying. ", runningInfo.getAddress().toString()), e);
                        }
                    } else {
                        logger.error(String.format("dump address %s has an error, retrying. caused by ", runningInfo.getAddress().toString()), e);
                        sendAlarm(destination, ExceptionUtils.getFullStackTrace(e));
                    }
                } finally {
                    // 重新置为中断状态
                    Thread.interrupted();
                    // 关闭一下链接
                    afterDump(erosaConnection);
                    try {
                        if (erosaConnection != null) {
                            erosaConnection.disconnect();
                        }
                    } catch (IOException e1) {
                        if (!running) {
                            throw new CanalParseException(String.format("disconnect address %s has an error, retrying. ", runningInfo.getAddress().toString()), e1);
                        } else {
                            logger.error("disconnect address {} has an error, retrying., caused by ", runningInfo.getAddress().toString(), e1);
                        }
                    }
                }
                // 出异常了，退出sink消费，释放一下状态
                eventSink.interrupt();
                // 重置一下缓冲队列，重新记录数据
                transactionBuffer.reset();
                // 重新置位
                binlogParser.reset();
                if (running) {
                    // sleep一段时间再进行重试
                    try {
                        Thread.sleep(10000 + RandomUtils.nextInt(10000));
                    } catch (InterruptedException e) {
                    }
                }
            }
            MDC.remove("destination");
        }
    });
    parseThread.setUncaughtExceptionHandler(handler);
    parseThread.setName(String.format("destination = %s , address = %s , EventParser", destination, runningInfo == null ? null : runningInfo.getAddress().toString()));
    parseThread.start();
}

Example 4 with Entry

use of com.alibaba.otter.canal.protocol.CanalEntry.Entry in project canal by alibaba.

the class EventTransactionBufferTest method testForceFlush.

@Test
public void testForceFlush() {
    final int bufferSize = 64;
    EventTransactionBuffer buffer = new EventTransactionBuffer();
    buffer.setBufferSize(bufferSize);
    buffer.setFlushCallback(new TransactionFlushCallback() {

        public void flush(List<Entry> transaction) throws InterruptedException {
            Assert.assertEquals(bufferSize, transaction.size());
            System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
            for (Entry data : transaction) {
                CanalEntry.Header header = data.getHeader();
                Date date = new Date(header.getExecuteTime());
                SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
                if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
                    // System.out.println(MessageFormat.format(messgae, new
                    // Object[] {
                    // Thread.currentThread().getName(),
                    // header.getLogfilename(), header.getLogfileoffset(),
                    // format.format(date),
                    // data.getEntry().getEntryType(), "" }));
                    System.out.println(data.getEntryType());
                } else {
                    System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(), header.getLogfileName(), header.getLogfileOffset(), format.format(date), header.getSchemaName(), header.getTableName() }));
                }
            }
            System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
        }
    });
    buffer.start();
    try {
        for (int i = 0; i < bufferSize * 2 + 1; i++) {
            buffer.add(buildEntry("1", 1L + i, 40L + i));
        }
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    buffer.stop();
}

Example 5 with Entry

use of com.alibaba.otter.canal.protocol.CanalEntry.Entry in project canal by alibaba.

the class LocalBinlogDumpTest method testSimple.

@Test
public void testSimple() {
    String directory = "/home/jianghang/tmp/binlog";
    final LocalBinlogEventParser controller = new LocalBinlogEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000006", 4L);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
    controller.setConnectionCharset(Charset.forName("UTF-8"));
    controller.setDirectory(directory);
    controller.setMasterPosition(startPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
                    continue;
                }
                if (entry.getEntryType() == EntryType.ROWDATA) {
                    RowChange rowChage = null;
                    try {
                        rowChage = RowChange.parseFrom(entry.getStoreValue());
                    } catch (Exception e) {
                        throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                    }
                    EventType eventType = rowChage.getEventType();
                    System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                    for (RowData rowData : rowChage.getRowDatasList()) {
                        if (eventType == EventType.DELETE) {
                            print(rowData.getBeforeColumnsList());
                        } else if (eventType == EventType.INSERT) {
                            print(rowData.getAfterColumnsList());
                        } else {
                            System.out.println("-------> before");
                            print(rowData.getBeforeColumnsList());
                            System.out.println("-------> after");
                            print(rowData.getAfterColumnsList());
                        }
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractCanalLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}