Examples with Event com.alibaba.otter.canal.store.model.Event used on opensource projects

Example 1 with Event

use of com.alibaba.otter.canal.store.model.Event in project canal by alibaba.

the class DummyEventStore method put.

public void put(List<Event> datas) throws InterruptedException, CanalStoreException {
    for (Event data : datas) {
        CanalEntry.Header header = data.getEntry().getHeader();
        Date date = new Date(header.getExecuteTime());
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
            // System.out.println(MessageFormat.format(messgae, new Object[]
            // { Thread.currentThread().getName(),
            // header.getLogfilename(), header.getLogfileoffset(),
            // format.format(date),
            // data.getEntry().getEntryType(), "" }));
            System.out.println(data.getEntry().getEntryType());
        } else {
            System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(), header.getLogfileName(), String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(), header.getSchemaName(), header.getTableName() }));
        }
    }
}

Example 2 with Event

use of com.alibaba.otter.canal.store.model.Event in project canal by alibaba.

the class CanalServerWithEmbedded method getWithoutAck.

/**
     * 不指定 position 获取事件。canal 会记住此 client 最新的 position。 <br/>
     * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
     * 
     * <pre>
     * 几种case:
     * a. 如果timeout为null，则采用tryGet方式，即时获取
     * b. 如果timeout不为null
     *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
     *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
     *    
     * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
     * </pre>
     */
@Override
public Message getWithoutAck(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    checkSubscribe(clientIdentity);
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    synchronized (canalInstance) {
        // 获取到流式数据中的最后一批获取的位置
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
        Events<Event> events = null;
        if (positionRanges != null) {
            // 存在流数据
            events = getEvents(canalInstance.getEventStore(), positionRanges.getStart(), batchSize, timeout, unit);
        } else {
            // ack后第一次获取
            Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
            if (start == null) {
                // 第一次，还没有过ack记录，则获取当前store中的第一条
                start = canalInstance.getEventStore().getFirstPosition();
            }
            events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
        }
        if (CollectionUtils.isEmpty(events.getEvents())) {
            logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result is null", new Object[] { clientIdentity.getClientId(), batchSize });
            // 返回空包，避免生成batchId，浪费性能
            return new Message(-1, new ArrayList<Entry>());
        } else {
            // 记录到流式信息
            Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
            List<Entry> entrys = Lists.transform(events.getEvents(), new Function<Event, Entry>() {

                public Entry apply(Event input) {
                    return input.getEntry();
                }
            });
            logger.info("getWithoutAck successfully, clientId:{} batchSize:{}  real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), batchSize, entrys.size(), batchId, events.getPositionRange());
            return new Message(batchId, entrys);
        }
    }
}

Example 3 with Event

use of com.alibaba.otter.canal.store.model.Event in project canal by alibaba.

the class MemoryEventStoreWithBuffer method cleanUntil.

public void cleanUntil(Position position) throws CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lock();
    try {
        long sequence = ackSequence.get();
        long maxSequence = getSequence.get();
        boolean hasMatch = false;
        long memsize = 0;
        for (long next = sequence + 1; next <= maxSequence; next++) {
            Event event = entries[getIndex(next)];
            memsize += calculateSize(event);
            boolean match = CanalEventUtils.checkPosition(event, (LogPosition) position);
            if (match) {
                // 找到对应的position，更新ack seq
                hasMatch = true;
                if (batchMode.isMemSize()) {
                    ackMemSize.addAndGet(memsize);
                    // 尝试清空buffer中的内存，将ack之前的内存全部释放掉
                    for (long index = sequence + 1; index < next; index++) {
                        // 设置为null
                        entries[getIndex(index)] = null;
                    }
                }
                if (ackSequence.compareAndSet(sequence, next)) {
                    // 避免并发ack
                    notFull.signal();
                    return;
                }
            }
        }
        if (!hasMatch) {
            // 找不到对应需要ack的position
            throw new CanalStoreException("no match ack position" + position.toString());
        }
    } finally {
        lock.unlock();
    }
}

Example 4 with Event

use of com.alibaba.otter.canal.store.model.Event in project canal by alibaba.

the class MemoryEventStoreWithBuffer method doGet.

private Events<Event> doGet(Position start, int batchSize) throws CanalStoreException {
    LogPosition startPosition = (LogPosition) start;
    long current = getSequence.get();
    long maxAbleSequence = putSequence.get();
    long next = current;
    long end = current;
    // 如果startPosition为null，说明是第一次，默认+1处理
    if (startPosition == null || !startPosition.getPostion().isIncluded()) {
        // 第一次订阅之后，需要包含一下start位置，防止丢失第一条记录
        next = next + 1;
    }
    if (current >= maxAbleSequence) {
        return new Events<Event>();
    }
    Events<Event> result = new Events<Event>();
    List<Event> entrys = result.getEvents();
    long memsize = 0;
    if (batchMode.isItemSize()) {
        end = (next + batchSize - 1) < maxAbleSequence ? (next + batchSize - 1) : maxAbleSequence;
        // 提取数据并返回
        for (; next <= end; next++) {
            Event event = entries[getIndex(next)];
            if (ddlIsolation && isDdl(event.getEntry().getHeader().getEventType())) {
                // 如果是ddl隔离，直接返回
                if (entrys.size() == 0) {
                    // 如果没有DML事件，加入当前的DDL事件
                    entrys.add(event);
                    // 更新end为当前
                    end = next;
                } else {
                    // 如果之前已经有DML事件，直接返回了，因为不包含当前next这记录，需要回退一个位置
                    // next-1一定大于current，不需要判断
                    end = next - 1;
                }
                break;
            } else {
                entrys.add(event);
            }
        }
    } else {
        long maxMemSize = batchSize * bufferMemUnit;
        for (; memsize <= maxMemSize && next <= maxAbleSequence; next++) {
            // 永远保证可以取出第一条的记录，避免死锁
            Event event = entries[getIndex(next)];
            if (ddlIsolation && isDdl(event.getEntry().getHeader().getEventType())) {
                // 如果是ddl隔离，直接返回
                if (entrys.size() == 0) {
                    // 如果没有DML事件，加入当前的DDL事件
                    entrys.add(event);
                    // 更新end为当前
                    end = next;
                } else {
                    // 如果之前已经有DML事件，直接返回了，因为不包含当前next这记录，需要回退一个位置
                    // next-1一定大于current，不需要判断
                    end = next - 1;
                }
                break;
            } else {
                entrys.add(event);
                memsize += calculateSize(event);
                // 记录end位点
                end = next;
            }
        }
    }
    PositionRange<LogPosition> range = new PositionRange<LogPosition>();
    result.setPositionRange(range);
    range.setStart(CanalEventUtils.createPosition(entrys.get(0)));
    range.setEnd(CanalEventUtils.createPosition(entrys.get(result.getEvents().size() - 1)));
    for (int i = entrys.size() - 1; i >= 0; i--) {
        Event event = entrys.get(i);
        if (CanalEntry.EntryType.TRANSACTIONBEGIN == event.getEntry().getEntryType() || CanalEntry.EntryType.TRANSACTIONEND == event.getEntry().getEntryType() || isDdl(event.getEntry().getHeader().getEventType())) {
            // 将事务头/尾设置可被为ack的点
            range.setAck(CanalEventUtils.createPosition(event));
            break;
        }
    }
    if (getSequence.compareAndSet(current, end)) {
        getMemSize.addAndGet(memsize);
        notFull.signal();
        return result;
    } else {
        return new Events<Event>();
    }
}

Example 5 with Event

use of com.alibaba.otter.canal.store.model.Event in project canal by alibaba.

the class DummyEventStore method put.

public void put(List<Event> datas) throws InterruptedException, CanalStoreException {
    Event data = datas.get(0);
    System.out.println("time:" + data.getEntry().getHeader().getExecuteTime());
}