Examples with MemoryEventStoreWithBuffer com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer used on opensource projects

Example 1 with MemoryEventStoreWithBuffer

use of com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer in project canal by alibaba.

the class CanalInstanceWithManager method initEventStore.

protected void initEventStore() {
    logger.info("init eventStore begin...");
    StorageMode mode = parameters.getStorageMode();
    if (mode.isMemory()) {
        MemoryEventStoreWithBuffer memoryEventStore = new MemoryEventStoreWithBuffer();
        memoryEventStore.setBufferSize(parameters.getMemoryStorageBufferSize());
        memoryEventStore.setBufferMemUnit(parameters.getMemoryStorageBufferMemUnit());
        memoryEventStore.setBatchMode(BatchMode.valueOf(parameters.getStorageBatchMode().name()));
        memoryEventStore.setDdlIsolation(parameters.getDdlIsolation());
        eventStore = memoryEventStore;
    } else if (mode.isFile()) {
        // 后续版本支持
        throw new CanalException("unsupport MetaMode for " + mode);
    } else if (mode.isMixed()) {
        // 后续版本支持
        throw new CanalException("unsupport MetaMode for " + mode);
    } else {
        throw new CanalException("unsupport MetaMode for " + mode);
    }
    if (eventStore instanceof AbstractCanalStoreScavenge) {
        StorageScavengeMode scavengeMode = parameters.getStorageScavengeMode();
        AbstractCanalStoreScavenge eventScavengeStore = (AbstractCanalStoreScavenge) eventStore;
        eventScavengeStore.setDestination(destination);
        eventScavengeStore.setCanalMetaManager(metaManager);
        eventScavengeStore.setOnAck(scavengeMode.isOnAck());
        eventScavengeStore.setOnFull(scavengeMode.isOnFull());
        eventScavengeStore.setOnSchedule(scavengeMode.isOnSchedule());
        if (scavengeMode.isOnSchedule()) {
            eventScavengeStore.setScavengeSchedule(parameters.getScavengeSchdule());
        }
    }
    logger.info("init eventStore end! \n\t load CanalEventStore:{}", eventStore.getClass().getName());
}

Example 2 with MemoryEventStoreWithBuffer

use of com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer in project canal by alibaba.

the class MemoryEventStoreMemBatchTest method testFullPut.

@Test
public void testFullPut() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();
    for (int i = 0; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        Assert.assertTrue(result);
    }
    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + bufferSize));
    Assert.assertFalse(result);
    try {
        result = eventStore.put(buildEvent("1", 1L, 1L + bufferSize), 1000L, TimeUnit.MILLISECONDS);
    } catch (CanalStoreException e) {
        Assert.fail(e.getMessage());
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertFalse(result);
    eventStore.stop();
}

Example 3 with MemoryEventStoreWithBuffer

use of com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer in project canal by alibaba.

the class MemoryEventStoreMemBatchTest method testAck.

@Test
public void testAck() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();
    for (int i = 0; i < bufferSize / 2; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    sleep(50L);
    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize / 2 - 1)));
    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());
    eventStore.cleanUntil(entrys1.getPositionRange().getEnd());
    sleep(50L);
    // 继续造数据
    for (int i = bufferSize / 2; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    Events<Event> entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
    System.out.println("second get size : " + entrys2.getEvents().size());
    eventStore.rollback();
    entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
    System.out.println("after rollback get size : " + entrys2.getEvents().size());
    first = eventStore.getFirstPosition();
    lastest = eventStore.getLatestPosition();
    List<Event> entrys = new ArrayList<Event>(entrys2.getEvents());
    Assert.assertEquals(first, entrys2.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys2.getPositionRange().getEnd());
    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys.get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys.get(entrys.size() - 1)));
    // 全部ack掉
    eventStore.cleanUntil(entrys2.getPositionRange().getEnd());
    // 最后就拿不到数据
    Events<Event> entrys3 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
    System.out.println("third get size : " + entrys3.getEvents().size());
    Assert.assertEquals(0, entrys3.getEvents().size());
    eventStore.stop();
}

Example 4 with MemoryEventStoreWithBuffer

use of com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer in project canal by alibaba.

the class MemoryEventStoreMemBatchTest method testOnePutOneGet.

@Test
public void testOnePutOneGet() {
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();
    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
    Assert.assertTrue(result);
    Position position = eventStore.getFirstPosition();
    Events<Event> entrys = eventStore.tryGet(position, 1);
    Assert.assertTrue(entrys.getEvents().size() == 1);
    Assert.assertEquals(position, entrys.getPositionRange().getStart());
    Assert.assertEquals(position, entrys.getPositionRange().getEnd());
    eventStore.stop();
}

Example 5 with MemoryEventStoreWithBuffer

use of com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer in project canal by alibaba.

the class MemoryEventStoreMemBatchTest method testFullPutBatchGet.

@Test
public void testFullPutBatchGet() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();
    for (int i = 0; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));
    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());
    Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
    Assert.assertEquals(first, entrys1.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());
    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
    eventStore.stop();
}