Examples with Message com.alibaba.otter.canal.protocol.Message used on opensource projects

Example 16 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class CanalTCPConsumer method getMessage.

@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
    try {
        Message message = canalConnector.getWithoutAck(batchSize, timeout, unit);
        long batchId = message.getId();
        currentBatchId = batchId;
        int size = message.getEntries().size();
        if (batchId == -1 || size == 0) {
            return null;
        } else {
            return MessageUtil.convert(message);
        }
    } catch (Throwable e) {
        throw new RuntimeException(e);
    }
}

Example 17 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class CanalServerWithEmbedded method getWithoutAck.

/**
 * 不指定 position 获取事件。canal 会记住此 client 最新的 position。 <br/>
 * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
 *
 * <pre>
 * 几种case:
 * a. 如果timeout为null，则采用tryGet方式，即时获取
 * b. 如果timeout不为null
 *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
 *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
 *
 * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
 * </pre>
 */
@Override
public Message getWithoutAck(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    checkSubscribe(clientIdentity);
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    synchronized (canalInstance) {
        // 获取到流式数据中的最后一批获取的位置
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
        Events<Event> events = null;
        if (positionRanges != null) {
            // 存在流数据
            events = getEvents(canalInstance.getEventStore(), positionRanges.getStart(), batchSize, timeout, unit);
        } else {
            // ack后第一次获取
            Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
            if (start == null) {
                // 第一次，还没有过ack记录，则获取当前store中的第一条
                start = canalInstance.getEventStore().getFirstPosition();
            }
            events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
        }
        if (CollectionUtils.isEmpty(events.getEvents())) {
            // 返回空包，避免生成batchId，浪费性能
            return new Message(-1, true, new ArrayList());
        } else {
            // 记录到流式信息
            Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
            boolean raw = isRaw(canalInstance.getEventStore());
            List entrys = null;
            if (raw) {
                entrys = Lists.transform(events.getEvents(), Event::getRawEntry);
            } else {
                entrys = Lists.transform(events.getEvents(), Event::getEntry);
            }
            if (logger.isInfoEnabled()) {
                logger.info("getWithoutAck successfully, clientId:{} batchSize:{}  real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), batchSize, entrys.size(), batchId, events.getPositionRange());
            }
            return new Message(batchId, raw, entrys);
        }
    }
}

Example 18 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class BaseCanalServerWithEmbededTest method testGet.

@Test
public void testGet() {
    int maxEmptyCount = 10;
    int emptyCount = 0;
    int totalCount = 0;
    server.subscribe(clientIdentity);
    while (emptyCount < maxEmptyCount) {
        Message message = server.get(clientIdentity, 11);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }
            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
        }
    }
    System.out.println("!!!!!! testGet totalCount : " + totalCount);
    server.unsubscribe(clientIdentity);
}

Example 19 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class BaseCanalServerWithEmbededTest method testRollback.

// @Test
public void testRollback() {
    int maxEmptyCount = 10;
    int emptyCount = 0;
    int totalCount = 0;
    server.subscribe(clientIdentity);
    while (emptyCount < maxEmptyCount) {
        Message message = server.getWithoutAck(clientIdentity, 11);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }
            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
        }
    }
    System.out.println("!!!!!! testRollback totalCount : " + totalCount);
    // 直接rollback掉，再取一次
    server.rollback(clientIdentity);
    emptyCount = 0;
    totalCount = 0;
    while (emptyCount < maxEmptyCount) {
        Message message = server.getWithoutAck(clientIdentity, 11);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }
            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
        }
    }
    System.out.println("!!!!!! testRollback after rollback ,  totalCount : " + totalCount);
    server.unsubscribe(clientIdentity);
}

Example 20 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class BaseCanalServerWithEmbededTest method testGetWithoutAck.

@Test
public void testGetWithoutAck() {
    int maxEmptyCount = 10;
    int emptyCount = 0;
    int totalCount = 0;
    server.subscribe(clientIdentity);
    while (emptyCount < maxEmptyCount) {
        Message message = server.getWithoutAck(clientIdentity, 11);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }
            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
            server.ack(clientIdentity, message.getId());
        }
    }
    System.out.println("!!!!!! testGetWithoutAck totalCount : " + totalCount);
    server.unsubscribe(clientIdentity);
}