Examples with Event com.alibaba.otter.canal.store.model.Event used on opensource projects

Example 16 with Event

use of com.alibaba.otter.canal.store.model.Event in project otter by alibaba.

the class OtterDownStreamHandlerIntergration method buildEvent.

private Event buildEvent() {
    Event event = new Event();
    event.setLogIdentity(new LogIdentity());
    Header.Builder headBuilder = Header.newBuilder();
    headBuilder.setEventLength(1000L);
    headBuilder.setExecuteTime(new Date().getTime());
    headBuilder.setLogfileName("mysql-bin.000001");
    headBuilder.setLogfileOffset(1000L);
    headBuilder.setSchemaName("test");
    headBuilder.setTableName("ljh");
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headBuilder.build());
    entryBuilder.setEntryType(EntryType.ROWDATA);
    RowChange.Builder rowChangeBuilder = RowChange.newBuilder();
    RowData.Builder rowDataBuilder = RowData.newBuilder();
    rowChangeBuilder.addRowDatas(rowDataBuilder.build());
    entryBuilder.setStoreValue(rowChangeBuilder.build().toByteString());
    Entry entry = entryBuilder.build();
    event.setEntry(entry);
    return event;
}

Example 17 with Event

use of com.alibaba.otter.canal.store.model.Event in project canal by alibaba.

the class DummyEventStore method tryPut.

public boolean tryPut(List<Event> datas) throws CanalStoreException {
    System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
    for (Event data : datas) {
        CanalEntry.Header header = data.getEntry().getHeader();
        Date date = new Date(header.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntry().getEntryType());
        } else {
            System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(), header.getLogfileName(), String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(), header.getSchemaName(), header.getTableName() }));
        }
    }
    System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
    return true;
}

Example 18 with Event

use of com.alibaba.otter.canal.store.model.Event in project canal by alibaba.

the class DummyEventStore method put.

public boolean put(List<Event> datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
    for (Event data : datas) {
        CanalEntry.Header header = data.getEntry().getHeader();
        Date date = new Date(header.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntry().getEntryType());
        } else {
            System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(), header.getLogfileName(), String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(), header.getSchemaName(), header.getTableName() }));
        }
    }
    return true;
}

Example 19 with Event

use of com.alibaba.otter.canal.store.model.Event in project canal by alibaba.

the class CanalServerWithEmbedded method get.

/**
     * 获取数据，可以指定超时时间.
     * 
     * <pre>
     * 几种case:
     * a. 如果timeout为null，则采用tryGet方式，即时获取
     * b. 如果timeout不为null
     *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
     *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
     * 
     * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
     * </pre>
     */
@Override
public Message get(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    checkSubscribe(clientIdentity);
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    synchronized (canalInstance) {
        // 获取到流式数据中的最后一批获取的位置
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
        if (positionRanges != null) {
            throw new CanalServerException(String.format("clientId:%s has last batch:[%s] isn't ack , maybe loss data", clientIdentity.getClientId(), positionRanges));
        }
        Events<Event> events = null;
        Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
        events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
        if (CollectionUtils.isEmpty(events.getEvents())) {
            logger.debug("get successfully, clientId:{} batchSize:{} but result is null", new Object[] { clientIdentity.getClientId(), batchSize });
            // 返回空包，避免生成batchId，浪费性能
            return new Message(-1, new ArrayList<Entry>());
        } else {
            // 记录到流式信息
            Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
            List<Entry> entrys = Lists.transform(events.getEvents(), new Function<Event, Entry>() {

                public Entry apply(Event input) {
                    return input.getEntry();
                }
            });
            logger.info("get successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), batchSize, entrys.size(), batchId, events.getPositionRange());
            // 直接提交ack
            ack(clientIdentity, batchId);
            return new Message(batchId, entrys);
        }
    }
}

Example 20 with Event

use of com.alibaba.otter.canal.store.model.Event in project canal by alibaba.

the class EntryEventSink method sinkData.

private boolean sinkData(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress) throws InterruptedException {
    boolean hasRowData = false;
    boolean hasHeartBeat = false;
    List<Event> events = new ArrayList<Event>();
    for (CanalEntry.Entry entry : entrys) {
        Event event = new Event(new LogIdentity(remoteAddress, -1L), entry);
        if (!doFilter(event)) {
            continue;
        }
        events.add(event);
        hasRowData |= (entry.getEntryType() == EntryType.ROWDATA);
        hasHeartBeat |= (entry.getEntryType() == EntryType.HEARTBEAT);
    }
    if (hasRowData) {
        // 存在row记录
        return doSink(events);
    } else if (hasHeartBeat) {
        // 存在heartbeat记录，直接跳给后续处理
        return doSink(events);
    } else {
        // 需要过滤的数据
        if (filterEmtryTransactionEntry && !CollectionUtils.isEmpty(events)) {
            long currentTimestamp = events.get(0).getEntry().getHeader().getExecuteTime();
            // 基于一定的策略控制，放过空的事务头和尾，便于及时更新数据库位点，表明工作正常
            if (Math.abs(currentTimestamp - lastEmptyTransactionTimestamp) > emptyTransactionInterval || lastEmptyTransactionCount.incrementAndGet() > emptyTransctionThresold) {
                lastEmptyTransactionCount.set(0L);
                lastEmptyTransactionTimestamp = currentTimestamp;
                return doSink(events);
            }
        }
        // 直接返回true，忽略空的事务头和尾
        return true;
    }
}