Examples with Position com.alibaba.otter.canal.protocol.position.Position used on opensource projects

Example 1 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(zooKeeperMetaManager);
    if (!zooKeeperMetaManager.isStart()) {
        zooKeeperMetaManager.start();
    }
    executor = Executors.newFixedThreadPool(1);
    destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {

        public List<ClientIdentity> apply(String destination) {
            return zooKeeperMetaManager.listAllSubscribeInfo(destination);
        }
    });
    cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {

        public Position apply(ClientIdentity clientIdentity) {
            Position position = zooKeeperMetaManager.getCursor(clientIdentity);
            if (position == null) {
                // 返回一个空对象标识，避免出现异常
                return nullCursor;
            } else {
                return position;
            }
        }
    });
    batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {

        public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
            // 读取一下zookeeper信息，初始化一次
            MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
            Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
            for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
                // 添加记录到指定batchId
                batches.addPositionRange(entry.getValue(), entry.getKey());
            }
            return batches;
        }
    });
}

Example 2 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MemoryEventStoreMemBatchTest method testAck.

@Test
public void testAck() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();
    for (int i = 0; i < bufferSize / 2; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    sleep(50L);
    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize / 2 - 1)));
    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());
    eventStore.cleanUntil(entrys1.getPositionRange().getEnd());
    sleep(50L);
    // 继续造数据
    for (int i = bufferSize / 2; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    Events<Event> entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
    System.out.println("second get size : " + entrys2.getEvents().size());
    eventStore.rollback();
    entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
    System.out.println("after rollback get size : " + entrys2.getEvents().size());
    first = eventStore.getFirstPosition();
    lastest = eventStore.getLatestPosition();
    List<Event> entrys = new ArrayList<Event>(entrys2.getEvents());
    Assert.assertEquals(first, entrys2.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys2.getPositionRange().getEnd());
    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys.get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys.get(entrys.size() - 1)));
    // 全部ack掉
    eventStore.cleanUntil(entrys2.getPositionRange().getEnd());
    // 最后就拿不到数据
    Events<Event> entrys3 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
    System.out.println("third get size : " + entrys3.getEvents().size());
    Assert.assertEquals(0, entrys3.getEvents().size());
    eventStore.stop();
}

Example 3 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MemoryEventStoreMemBatchTest method testOnePutOneGet.

@Test
public void testOnePutOneGet() {
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();
    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
    Assert.assertTrue(result);
    Position position = eventStore.getFirstPosition();
    Events<Event> entrys = eventStore.tryGet(position, 1);
    Assert.assertTrue(entrys.getEvents().size() == 1);
    Assert.assertEquals(position, entrys.getPositionRange().getStart());
    Assert.assertEquals(position, entrys.getPositionRange().getEnd());
    eventStore.stop();
}

Example 4 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MemoryEventStoreMemBatchTest method testFullPutBatchGet.

@Test
public void testFullPutBatchGet() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();
    for (int i = 0; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));
    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());
    Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
    Assert.assertEquals(first, entrys1.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());
    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
    eventStore.stop();
}

Example 5 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MemoryEventStoreMemBatchTest method testBlockPutOneGet.

@Test
public void testBlockPutOneGet() {
    final MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(16);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();
    final int batchSize = 10;
    for (int i = 0; i < batchSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
        Assert.assertTrue(result);
    }
    final Position position = eventStore.getFirstPosition();
    try {
        Events<Event> entrys = eventStore.get(position, batchSize);
        Assert.assertTrue(entrys.getEvents().size() == batchSize);
        Assert.assertEquals(position, entrys.getPositionRange().getStart());
        Assert.assertEquals(position, entrys.getPositionRange().getEnd());
    } catch (CanalStoreException e) {
    } catch (InterruptedException e) {
    }
    ExecutorService executor = Executors.newFixedThreadPool(1);
    executor.submit(new Runnable() {

        public void run() {
            boolean result = false;
            try {
                eventStore.get(position, batchSize);
            } catch (CanalStoreException e) {
            } catch (InterruptedException e) {
                System.out.println("interrupt occured.");
                result = true;
            }
            Assert.assertTrue(result);
        }
    });
    try {
        Thread.sleep(1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    executor.shutdownNow();
    try {
        Thread.sleep(1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    eventStore.stop();
}