use of com.linkedin.databus.bootstrap.common.BootstrapReadOnlyConfig in project databus by linkedin.
the class TestBootstrapSCNProcessor method testShouldBypassSnapshot.
@Test
public void testShouldBypassSnapshot() throws SQLException, BootstrapProcessingException, NoSuchFieldException, IllegalAccessException, NoSuchMethodException {
BootstrapSCNProcessor bsp = new BootstrapSCNProcessor();
Long defaultRowsThresholdForSnapshotBypass = Long.MAX_VALUE;
Map<String, Long> rowsThresholdForSnapshotBypass = new HashMap<String, Long>();
Map<String, Boolean> disableSnapshotBypass = new HashMap<String, Boolean>();
boolean predicatePushDown = false;
Map<String, Boolean> predicatePushDownBypass = new HashMap<String, Boolean>();
int queryTimeoutInSec = 10;
BootstrapReadOnlyConfig db = null;
boolean enableMinScnCheck = false;
final long longestDbTxnTimeMins = 240L;
BootstrapServerStaticConfig bssc = new BootstrapServerStaticConfig(defaultRowsThresholdForSnapshotBypass, rowsThresholdForSnapshotBypass, disableSnapshotBypass, predicatePushDown, predicatePushDownBypass, queryTimeoutInSec, enableMinScnCheck, db, longestDbTxnTimeMins);
Field field = bsp.getClass().getDeclaredField("_config");
field.setAccessible(true);
field.set(bsp, bssc);
int srcId = 101;
long sinceScn = 5;
long startScn = 10;
BootstrapDBMetaDataDAO bmdd = EasyMock.createMock(BootstrapDBMetaDataDAO.class);
EasyMock.expect(bmdd.getLogIdToCatchup(srcId, startScn)).andReturn(0).anyTimes();
EasyMock.expect(bmdd.getLogIdToCatchup(srcId, sinceScn)).andReturn(0).anyTimes();
EasyMock.replay(bmdd);
Field dbDaoField = bsp.getClass().getDeclaredField("_dbDao");
dbDaoField.setAccessible(true);
dbDaoField.set(bsp, bmdd);
List<SourceStatusInfo> srcList = new ArrayList<SourceStatusInfo>();
String name = "foo";
SourceStatusInfo ssi = new SourceStatusInfo(name, srcId, 4);
srcList.add(ssi);
// case 1. Single source, defaultRowsThresholdForSnapshotBypass set to Long.MAX_VALUE,
// individual overrides not set
boolean sbs = bsp.shouldBypassSnapshot(sinceScn, startScn, srcList);
Assert.assertEquals(true, sbs);
// case 2. Single source, defaultRowsThresholdForSnapshotBypass set to finite value,
// individual overrides set for the source
rowsThresholdForSnapshotBypass.put(name, Long.MAX_VALUE);
BootstrapServerStaticConfig bssc2 = new BootstrapServerStaticConfig(defaultRowsThresholdForSnapshotBypass, rowsThresholdForSnapshotBypass, disableSnapshotBypass, predicatePushDown, predicatePushDownBypass, queryTimeoutInSec, enableMinScnCheck, db, longestDbTxnTimeMins);
field.set(bsp, bssc2);
sbs = bsp.shouldBypassSnapshot(sinceScn, startScn, srcList);
Assert.assertEquals(true, sbs);
// Case 3: Detect case when the log is not available on log tables ( so it should NOT bypass snapshot )
BootstrapDBMetaDataDAO bmdd2 = EasyMock.createMock(BootstrapDBMetaDataDAO.class);
EasyMock.expect(bmdd2.getLogIdToCatchup(srcId, startScn)).andReturn(2).anyTimes();
EasyMock.expect(bmdd2.getLogIdToCatchup(srcId, sinceScn)).andThrow(new BootstrapProcessingException(""));
EasyMock.replay(bmdd2);
Field dbDaoField2 = bsp.getClass().getDeclaredField("_dbDao");
dbDaoField2.setAccessible(true);
dbDaoField2.set(bsp, bmdd2);
sbs = bsp.shouldBypassSnapshot(sinceScn, startScn, srcList);
Assert.assertEquals(false, sbs);
}
use of com.linkedin.databus.bootstrap.common.BootstrapReadOnlyConfig in project databus by linkedin.
the class TestBootstrap method testBootstrapProcessor.
@Test
public void testBootstrapProcessor() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, BootstrapProcessingException, DatabusException, BootstrapDatabaseTooOldException, BootstrapDBException {
EventProcessor processorCallback = new EventProcessor();
BootstrapConfig config = new BootstrapConfig();
BootstrapReadOnlyConfig staticConfig = config.build();
BootstrapServerConfig configBuilder = new BootstrapServerConfig();
configBuilder.setEnableMinScnCheck(false);
BootstrapServerStaticConfig staticServerConfig = configBuilder.build();
BootstrapProcessor processor = new BootstrapProcessor(staticServerConfig, null);
String sourceName = "TestBootstrap.testBootstrapProcessor.events";
// Create the tables for all the sources before starting up the threads
BootstrapConn _bootstrapConn = new BootstrapConn();
boolean autoCommit = true;
_bootstrapConn.initBootstrapConn(autoCommit, staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBName());
BootstrapDBMetaDataDAO dao = new BootstrapDBMetaDataDAO(_bootstrapConn, staticConfig.getBootstrapDBHostname(), staticConfig.getBootstrapDBUsername(), staticConfig.getBootstrapDBPassword(), staticConfig.getBootstrapDBName(), autoCommit);
SourceStatusInfo srcStatusInfo = dao.getSrcIdStatusFromDB(sourceName, false);
if (srcStatusInfo.getSrcId() >= 0) {
dao.dropSourceInDB(srcStatusInfo.getSrcId());
}
dao.addNewSourceInDB(sourceName, BootstrapProducerStatus.ACTIVE);
srcStatusInfo = dao.getSrcIdStatusFromDB(sourceName, false);
setBootstrapLoginfoTable(_bootstrapConn, 0, Long.MAX_VALUE);
//insert one row
insertOneSnapshotEvent(dao, srcStatusInfo.getSrcId(), Long.MAX_VALUE - 10, Long.MAX_VALUE - 100, "check", "test_payload_data");
BootstrapCheckpointHandler bstCheckpointHandler = new BootstrapCheckpointHandler(sourceName);
Checkpoint c = bstCheckpointHandler.createInitialBootstrapCheckpoint(null, 0L);
c.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
c.setBootstrapStartScn(Long.MAX_VALUE - 10);
c.setSnapshotOffset(Long.MAX_VALUE - 1000);
// System.out.println("Snapshot");
processor.streamSnapShotRows(c, processorCallback);
Assert.assertEquals(processorCallback.getrIds().size(), 1);
Assert.assertEquals(Long.MAX_VALUE - 10, processorCallback.getrIds().get(0).longValue());
Assert.assertEquals(Long.MAX_VALUE - 100, processorCallback.getSequences().get(0).longValue());
Assert.assertEquals("check", processorCallback.getSrcKeys().get(0));
Assert.assertEquals("test_payload_data", new String(processorCallback.getValues().get(0)));
processorCallback.reset();
c.setSnapshotOffset(Checkpoint.FULLY_CONSUMED_WINDOW_OFFSET);
c.setBootstrapTargetScn(c.getBootstrapStartScn().longValue());
bstCheckpointHandler.advanceAfterTargetScn(c);
bstCheckpointHandler.advanceAfterSnapshotPhase(c);
// c.setConsumptionMode(DbusClientMode.BOOTSTRAP_CATCHUP);
// c.setCatchupSource(sourceName);
// c.setWindowScn(0L);
// c.setWindowOffset(0);
// System.out.println("Catchup");
boolean phaseCompleted = processor.streamCatchupRows(c, processorCallback);
Assert.assertEquals(true, phaseCompleted);
}
Aggregations