Examples with Position com.alibaba.otter.canal.protocol.position.Position used on opensource projects

Example 16 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class AbstractMetaManagerTest method doCursorTest.

public Position doCursorTest(CanalMetaManager metaManager) {
    metaManager.subscribe(clientIdentity);
    Position position1 = metaManager.getCursor(clientIdentity);
    Assert.assertNull(position1);
    PositionRange range = buildRange(1);
    metaManager.updateCursor(clientIdentity, range.getStart());
    Position position2 = metaManager.getCursor(clientIdentity);
    Assert.assertEquals(range.getStart(), position2);
    metaManager.updateCursor(clientIdentity, range.getEnd());
    Position position3 = metaManager.getCursor(clientIdentity);
    Assert.assertEquals(range.getEnd(), position3);
    return position3;
}

Example 17 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class FileMixedMetaManagerTest method testCursorAll.

@Test
public void testCursorAll() {
    FileMixedMetaManager metaManager = new FileMixedMetaManager();
    metaManager.setDataDir(dataDir);
    metaManager.setPeriod(100);
    metaManager.start();
    Position lastPosition = doCursorTest(metaManager);
    sleep(1000L);
    // 重新构建一次，能获得上一次zk上的记录
    FileMixedMetaManager metaManager2 = new FileMixedMetaManager();
    metaManager2.setDataDir(dataDir);
    metaManager2.setPeriod(100);
    metaManager2.start();
    Position position = metaManager2.getCursor(clientIdentity);
    Assert.assertEquals(position, lastPosition);
    metaManager.stop();
}

Example 18 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MixedMetaManagerTest method testCursorAll.

@Test
public void testCursorAll() {
    MixedMetaManager metaManager = new MixedMetaManager();
    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);
    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();
    Position lastPosition = doCursorTest(metaManager);
    sleep(1000L);
    // 重新构建一次，能获得上一次zk上的记录
    MixedMetaManager metaManager2 = new MixedMetaManager();
    metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager2.start();
    Position position = metaManager2.getCursor(clientIdentity);
    Assert.assertEquals(position, lastPosition);
    metaManager.stop();
}

Example 19 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class CanalServerWithEmbedded method get.

/**
     * 获取数据，可以指定超时时间.
     * 
     * <pre>
     * 几种case:
     * a. 如果timeout为null，则采用tryGet方式，即时获取
     * b. 如果timeout不为null
     *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
     *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
     * 
     * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
     * </pre>
     */
@Override
public Message get(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    checkSubscribe(clientIdentity);
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    synchronized (canalInstance) {
        // 获取到流式数据中的最后一批获取的位置
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
        if (positionRanges != null) {
            throw new CanalServerException(String.format("clientId:%s has last batch:[%s] isn't ack , maybe loss data", clientIdentity.getClientId(), positionRanges));
        }
        Events<Event> events = null;
        Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
        events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
        if (CollectionUtils.isEmpty(events.getEvents())) {
            logger.debug("get successfully, clientId:{} batchSize:{} but result is null", new Object[] { clientIdentity.getClientId(), batchSize });
            // 返回空包，避免生成batchId，浪费性能
            return new Message(-1, new ArrayList<Entry>());
        } else {
            // 记录到流式信息
            Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
            List<Entry> entrys = Lists.transform(events.getEvents(), new Function<Event, Entry>() {

                public Entry apply(Event input) {
                    return input.getEntry();
                }
            });
            logger.info("get successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), batchSize, entrys.size(), batchId, events.getPositionRange());
            // 直接提交ack
            ack(clientIdentity, batchId);
            return new Message(batchId, entrys);
        }
    }
}

Example 20 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MemoryEventStoreMemBatchTest method testRollback.

@Test
public void testRollback() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();
    for (int i = 0; i < bufferSize / 2; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    sleep(50L);
    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize / 2 - 1)));
    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());
    eventStore.rollback();
    entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("after rollback get size : " + entrys1.getEvents().size());
    Assert.assertTrue(entrys1.getEvents().size() == bufferSize / 2);
    // 继续造数据
    for (int i = bufferSize / 2; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    Events<Event> entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
    System.out.println("second get size : " + entrys2.getEvents().size());
    eventStore.rollback();
    entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
    System.out.println("after rollback get size : " + entrys2.getEvents().size());
    Assert.assertTrue(entrys2.getEvents().size() == bufferSize);
    first = eventStore.getFirstPosition();
    lastest = eventStore.getLatestPosition();
    List<Event> entrys = new ArrayList<Event>(entrys2.getEvents());
    Assert.assertTrue(entrys.size() == bufferSize);
    Assert.assertEquals(first, entrys2.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys2.getPositionRange().getEnd());
    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys.get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys.get(bufferSize - 1)));
    eventStore.stop();
}