use of com.linkedin.databus.core.BootstrapCheckpointHandler in project databus by linkedin.
the class TestGenericDispatcher method testBootstrapPartialWindowScnOrdering.
@Test
public void testBootstrapPartialWindowScnOrdering() throws Exception {
final Logger log = Logger.getLogger("TestGenericDispatcher.testBootstrapPartialWindowScnOrdering");
//log.setLevel(Level.DEBUG);
log.info("start");
//DDSDBUS-1889: Ensure bootstrap onCheckpoint() callback receives bootstrapSinceScn - not some scn.
int numEvents = 100;
int maxWindowSize = 25;
/* Experiment setup */
int payloadSize = 20;
int numCheckpoints = numEvents / maxWindowSize;
/* Consumer creation */
//setup consumer to fail on end of first full window
int timeTakenForDataEventInMs = 1;
int timeTakenForControlEventInMs = 1;
int numFailCheckpointEvent = 0;
int numFailDataEvent = 0;
int numFailEndWindow = 1;
int numFailures = 1;
//fail at the specified window; no retries; so the dispatcher should stop having written one window; but having checkpointed the other
//thanks to a very small checkpoint frequency threshold
TimeoutTestConsumer tConsumer = new TimeoutTestConsumer(timeTakenForDataEventInMs, timeTakenForControlEventInMs, numFailCheckpointEvent, numFailDataEvent, numFailEndWindow, numFailures);
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
short srcId = 1;
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, srcId, SOURCE1_SCHEMA_STR));
schemaMap.put(1L, l1);
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
List<String> sources = new ArrayList<String>();
for (int i = 1; i <= 1; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
long consumerTimeBudgetMs = 60 * 1000;
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(tConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
//Single threaded execution of consumer
MultiConsumerCallback mConsumer = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(1), consumerTimeBudgetMs, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
/* Generate events **/
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add(srcId);
DbusEventGenerator evGen = new DbusEventGenerator(15000, srcIdList);
Assert.assertTrue(evGen.generateEvents(numEvents, maxWindowSize, 512, payloadSize, true, srcTestEvents) > 0);
int totalSize = 0;
int maxSize = 0;
for (DbusEvent e : srcTestEvents) {
totalSize += e.size();
maxSize = (e.size() > maxSize) ? e.size() : maxSize;
}
/* Source configuration */
double thresholdChkptPct = 5.0;
DatabusSourcesConnection.Config conf = new DatabusSourcesConnection.Config();
conf.setCheckpointThresholdPct(thresholdChkptPct);
conf.getDispatcherRetries().setMaxRetryNum(0);
conf.setFreeBufferThreshold(maxSize);
conf.setConsumerTimeBudgetMs(consumerTimeBudgetMs);
int freeBufferThreshold = conf.getFreeBufferThreshold();
DatabusSourcesConnection.StaticConfig connConfig = conf.build();
//make buffer large enough to hold data; the control events are large that contain checkpoints
int producerBufferSize = totalSize * 2 + numCheckpoints * 10 * maxSize * 5 + freeBufferThreshold;
int individualBufferSize = producerBufferSize;
int indexSize = producerBufferSize / 10;
int stagingBufferSize = producerBufferSize;
/*Event Buffer creation */
TestGenericDispatcherEventBuffer dataEventsBuffer = new TestGenericDispatcherEventBuffer(getConfig(producerBufferSize, individualBufferSize, indexSize, stagingBufferSize, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE));
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
/* Generic Dispatcher creation */
InMemoryPersistenceProvider cpPersister = new InMemoryPersistenceProvider();
BootstrapDispatcher dispatcher = new BootstrapDispatcher("bootstrapPartialWindowCheckpointPersistence", connConfig, subs, cpPersister, dataEventsBuffer, mConsumer, //relaypuller
null, //mbean server
null, //ClientImpl
null, //registrationId
null, // logger
null);
dispatcher.setSchemaIdCheck(false);
BootstrapCheckpointHandler cptHandler = new BootstrapCheckpointHandler("source1");
long sinceScn = 15000L;
long startTsNsecs = System.nanoTime();
final Checkpoint initCheckpoint = cptHandler.createInitialBootstrapCheckpoint(null, sinceScn);
initCheckpoint.setBootstrapStartNsecs(startTsNsecs);
initCheckpoint.setBootstrapStartScn(0L);
/* Launch writer */
//numBootstrapCheckpoint - number of checkpoints before writing end of period
int numBootstrapCheckpoint = 4;
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, dataEventsBuffer, numBootstrapCheckpoint, null);
//simulate bootstrap server; use this checkpoint as init checkpoint
eventProducer.setBootstrapCheckpoint(initCheckpoint);
Thread tEmitter = new Thread(eventProducer);
//be generous ; use worst case for num control events
long waitTimeMs = (numEvents * timeTakenForDataEventInMs + numEvents * timeTakenForControlEventInMs) * 4;
tEmitter.start();
tEmitter.join(waitTimeMs);
/* Launch dispatcher */
Thread tDispatcher = new Thread(dispatcher);
tDispatcher.start();
/* Now initialize this state machine */
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
//expect dispatcher to fail - at end of window
tDispatcher.join(waitTimeMs);
Assert.assertFalse(tEmitter.isAlive());
Assert.assertFalse(tDispatcher.isAlive());
LOG.info("tConsumer: " + tConsumer);
HashMap<List<String>, Checkpoint> cps = cpPersister.getCheckpoints();
Assert.assertTrue(cps.size() > 0);
for (Map.Entry<List<String>, Checkpoint> i : cps.entrySet()) {
Checkpoint cp = i.getValue();
LOG.info("checkpoint=" + cp);
Assert.assertEquals(cp.getConsumptionMode(), DbusClientMode.BOOTSTRAP_SNAPSHOT);
//check if progress has been made during bootstrap
Assert.assertTrue(cp.getSnapshotOffset() > 0);
//these two values should be unchanged during the course of bootstrap
Assert.assertEquals(sinceScn, cp.getBootstrapSinceScn().longValue());
Assert.assertEquals(startTsNsecs, cp.getBootstrapStartNsecs());
//the tsNsec normally udpdated by client at end of window should be a no-op during bootstrap
Assert.assertEquals(Checkpoint.UNSET_TS_NSECS, cp.getTsNsecs());
//the scn passed to consumers during onCheckpoint should be the sinceSCN and not any other interim value
Assert.assertEquals(cp.getBootstrapSinceScn().longValue(), tConsumer.getLastSeenCheckpointScn());
}
log.info("end\n");
}
Aggregations