Examples with CacheEntry com.linkedin.databus.client.pub.FileSystemCheckpointPersistenceProvider.CacheEntry used on opensource projects

Example 1 with CacheEntry

use of com.linkedin.databus.client.pub.FileSystemCheckpointPersistenceProvider.CacheEntry in project databus by linkedin.

the class TestFileSystemCheckpointPersistanceProvider method testCacheEntryloadCurrentCheckpoint_new.

@Test
public void testCacheEntryloadCurrentCheckpoint_new() throws Exception {
    File checkpointDir = new File("/tmp/databus2-checkpoints-test");
    FileSystemCheckpointPersistenceProvider.Config config = new FileSystemCheckpointPersistenceProvider.Config();
    config.setRootDirectory(checkpointDir.getAbsolutePath());
    config.getRuntime().setHistoryEnabled(false);
    FileSystemCheckpointPersistenceProvider checkpointProvider = new FileSystemCheckpointPersistenceProvider(config, 2);
    List<String> sourceNames = Arrays.asList("source1", "source2");
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sourceNames);
    List<String> subsList = checkpointProvider.convertSubsToListOfStrings(subs);
    String streamId = FileSystemCheckpointPersistenceProvider.calcStreamId(subsList);
    assertEquals("cp_source1-source2", streamId);
    File streamFile = new File(checkpointDir, streamId + ".current");
    if (streamFile.exists()) {
        assertTrue(streamFile.delete());
    }
    CacheEntry cacheEntry = checkpointProvider.new CacheEntry(streamId, null);
    Checkpoint checkpoint = cacheEntry.getCheckpoint();
    assertNull(checkpoint);
}

Example 2 with CacheEntry

use of com.linkedin.databus.client.pub.FileSystemCheckpointPersistenceProvider.CacheEntry in project databus by linkedin.

the class TestFileSystemCheckpointPersistanceProvider method testCacheEntry_store_load_withroll.

@Test
public void testCacheEntry_store_load_withroll() throws Exception {
    File checkpointDir = new File("/tmp/databus2-checkpoints-test");
    FileSystemCheckpointPersistenceProvider.Config config = new FileSystemCheckpointPersistenceProvider.Config();
    config.setRootDirectory(checkpointDir.getAbsolutePath());
    config.getRuntime().setHistoryEnabled(true);
    config.getRuntime().setHistorySize(10);
    FileSystemCheckpointPersistenceProvider checkpointProvider = new FileSystemCheckpointPersistenceProvider(config, 2);
    List<String> sourceNames = Arrays.asList("source1", "source2", "source3");
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sourceNames);
    List<String> subsList = checkpointProvider.convertSubsToListOfStrings(subs);
    String streamId = FileSystemCheckpointPersistenceProvider.calcStreamId(subsList);
    assertEquals("cp_source1-source2-source3", streamId);
    //clean up checkpoint files
    File checkpointFile = new File(checkpointDir, streamId + ".current");
    if (checkpointFile.exists()) {
        assertTrue(checkpointFile.delete());
    }
    for (int i = 0; i < 15; ++i) {
        File f = FileSystemCheckpointPersistenceProvider.StaticConfig.generateCheckpointFile(checkpointDir, streamId + ".", i);
        if (f.exists()) {
            assertTrue(f.delete());
        }
    }
    Checkpoint[] cp = new Checkpoint[15];
    //store checkpoints
    CacheEntry cacheEntry = checkpointProvider.new CacheEntry(streamId, null);
    for (int i = 0; i < 15; ++i) {
        cp[i] = new Checkpoint();
        cp[i].setWindowScn((long) i * i * i);
        cp[i].setWindowOffset(i * i);
        cp[i].setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
        assertTrue(cacheEntry.setCheckpoint(cp[i]));
        assertTrue(checkpointFile.exists());
        CacheEntry cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
        Checkpoint cp2 = cacheEntry2.getCheckpoint();
        assertNotNull(cp2);
        //TODO need to use a Checkpoint.equals() method
        assertEquals(cp[i].getConsumptionMode(), cp2.getConsumptionMode());
        assertEquals(cp[i].getWindowScn(), cp2.getWindowScn());
        assertEquals(cp[i].getPrevScn(), cp2.getPrevScn());
        assertEquals(cp[i].getWindowOffset(), cp2.getWindowOffset());
    }
    //make sure we don't go over history size
    for (int i = 10; i < 15; ++i) {
        File f = FileSystemCheckpointPersistenceProvider.StaticConfig.generateCheckpointFile(checkpointDir, streamId, i);
        assertTrue(!f.exists());
    }
    //check correctness of history
    for (int i = 0; i < 10; ++i) {
        assertTrue(checkpointFile.delete());
        File f = FileSystemCheckpointPersistenceProvider.StaticConfig.generateCheckpointFile(checkpointDir, streamId + ".", i);
        assertTrue(f.renameTo(checkpointFile));
        CacheEntry cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
        Checkpoint cp2 = cacheEntry2.getCheckpoint();
        assertNotNull(cp2);
        final int j = 13 - i;
        //TODO need to use a Checkpoint.equals() method
        assertEquals(cp[j].getConsumptionMode(), cp2.getConsumptionMode());
        assertEquals(cp[j].getWindowScn(), cp2.getWindowScn());
        assertEquals(cp[j].getPrevScn(), cp2.getPrevScn());
        assertEquals(cp[j].getWindowOffset(), cp2.getWindowOffset());
    }
}

Example 3 with CacheEntry

use of com.linkedin.databus.client.pub.FileSystemCheckpointPersistenceProvider.CacheEntry in project databus by linkedin.

the class TestFileSystemCheckpointPersistanceProvider method testCacheEntry_store_load_noroll.

@Test
public void testCacheEntry_store_load_noroll() throws Exception {
    File checkpointDir = new File("/tmp/databus2-checkpoints-test");
    FileSystemCheckpointPersistenceProvider.Config config = new FileSystemCheckpointPersistenceProvider.Config();
    config.setRootDirectory(checkpointDir.getAbsolutePath());
    config.getRuntime().setHistoryEnabled(false);
    FileSystemCheckpointPersistenceProvider checkpointProvider = new FileSystemCheckpointPersistenceProvider(config, 2);
    List<String> sourceNames = Arrays.asList("source1", "source2");
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sourceNames);
    List<String> subsList = checkpointProvider.convertSubsToListOfStrings(subs);
    String streamId = FileSystemCheckpointPersistenceProvider.calcStreamId(subsList);
    assertEquals("cp_source1-source2", streamId);
    File checkpointFile = new File(checkpointDir, streamId + ".current");
    if (checkpointFile.exists()) {
        assertTrue(checkpointFile.delete());
    }
    //simple checkpoint
    Checkpoint cp1 = new Checkpoint();
    cp1.setFlexible();
    CacheEntry cacheEntry = checkpointProvider.new CacheEntry(streamId, null);
    assertTrue(cacheEntry.setCheckpoint(cp1));
    assertTrue(checkpointFile.exists());
    CacheEntry cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
    Checkpoint cp2 = cacheEntry2.getCheckpoint();
    assertNotNull(cp2);
    //TODO need to use a Checkpoint.equals() method
    assertEquals(DbusClientMode.ONLINE_CONSUMPTION, cp2.getConsumptionMode());
    assertTrue(cp2.getFlexible());
    //more complex checkpoint plus overwriting current state
    Checkpoint cp3 = new Checkpoint();
    cp3.setWindowScn(1234L);
    cp3.setWindowOffset(9876);
    cp3.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    assertTrue(cacheEntry.setCheckpoint(cp3));
    assertTrue(checkpointFile.exists());
    File cpBackupFile = new File(checkpointDir, streamId + ".oldcurrent");
    assertTrue(cpBackupFile.exists());
    cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
    cp2 = cacheEntry2.getCheckpoint();
    assertNotNull(cp2);
    //TODO need to use a Checkpoint.equals() method
    assertEquals(cp3.getConsumptionMode(), cp2.getConsumptionMode());
    assertEquals(cp3.getWindowScn(), cp2.getWindowScn());
    assertEquals(cp3.getPrevScn(), cp2.getPrevScn());
    assertEquals(cp3.getWindowOffset(), cp2.getWindowOffset());
    //make sure the backup still works
    assertTrue(checkpointFile.delete());
    assertTrue(!checkpointFile.exists());
    if (!cpBackupFile.renameTo(checkpointFile))
        LOG.error("file rename failed: " + cpBackupFile.getAbsolutePath() + " --> " + checkpointFile.getAbsolutePath());
    assertTrue(checkpointFile.exists());
    cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
    cp2 = cacheEntry2.getCheckpoint();
    assertNotNull(cp2);
    //TODO need to use a Checkpoint.equals() method
    assertEquals(DbusClientMode.ONLINE_CONSUMPTION, cp2.getConsumptionMode());
    assertTrue(cp2.getFlexible());
    //try to keep some stuff around and see if things go south
    Checkpoint cp4 = new Checkpoint();
    cp4.setWindowScn(1111L);
    cp4.setWindowOffset(2222);
    cp4.setConsumptionMode(DbusClientMode.BOOTSTRAP_CATCHUP);
    File newCpFile = new File(checkpointDir, streamId + ".newcheckpoint");
    PrintWriter tmpWriter = new PrintWriter(newCpFile);
    try {
        tmpWriter.println("Dummy");
        assertTrue(cacheEntry.setCheckpoint(cp4));
        assertTrue(checkpointFile.exists());
        cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
        cp2 = cacheEntry2.getCheckpoint();
        assertNotNull(cp2);
        //TODO need to use a Checkpoint.equals() method
        assertEquals(cp4.getConsumptionMode(), cp2.getConsumptionMode());
        assertEquals(cp4.getWindowScn(), cp2.getWindowScn());
        assertEquals(cp4.getPrevScn(), cp2.getPrevScn());
        assertEquals(cp4.getWindowOffset(), cp2.getWindowOffset());
    } finally {
        tmpWriter.close();
    }
}