use of com.linkedin.databus.core.StreamEventsResult in project databus by linkedin.
the class TestGenericDispatcher method testPartialWindowRollback.
@Test(groups = { "small", "functional" })
public /**
* Tests the case where the dispatcher exits the main processing loop in {@link GenericDispatcher#doDispatchEvents()}
* with a partial window and the flushing of the outstanding callbacks fails. We want to make sure that a rollback
* is correctly triggered.
*
* The test simulates the following case: e1_1 e1_2 e1_3 <EOW> e2_1 e2_2 e2_3 <EOW> ... with a failure in the e2_2
* callback.
*
* 1) Read full first window: e1_1 e1_2 e1_3 <EOW>
* 2) Read partial second window: e2_1 e2_2
* 3) The above should fail -- verify that rollback is called
* 4) Read the rest
*/
void testPartialWindowRollback() throws Exception {
final Logger log = Logger.getLogger("TestGenericDispatcher.testPartialWindowRollback");
// log.setLevel(Level.INFO);
log.info("start");
final Level saveLevel = Logger.getLogger("com.linkedin.databus.client").getLevel();
// Logger.getLogger("com.linkedin.databus.client").setLevel(Level.DEBUG);
// generate events
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add((short) 1);
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
final int numEvents = 9;
// 1-based number of the event callback to fail
final int numOfFailureEvent = 5;
final int numEventsPerWindow = 3;
final int payloadSize = 200;
final int numWindows = (int) Math.ceil(1.0 * numEvents / numEventsPerWindow);
Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
// find out how much data we need to stream for the failure
// account for the EOW event which is < payload size
int win1Size = payloadSize - 1;
int win2Size = 0;
int eventN = 0;
for (DbusEvent e : srcTestEvents) {
eventN++;
if (eventN <= numEventsPerWindow) {
win1Size += e.size();
} else if (eventN <= numOfFailureEvent) {
win2Size += e.size();
}
}
// serialize the events to a buffer so they can be sent to the client
final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
// Create destination (client) buffer
final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
// Create dispatcher
final TimeoutTestConsumer mockConsumer = new TimeoutTestConsumer(100, 10, 0, numOfFailureEvent, 0, 1);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), 1000, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
dispatcher.setSchemaIdCheck(false);
Thread dispatcherThread = new Thread(dispatcher);
dispatcherThread.setDaemon(true);
log.info("starting dispatcher thread");
dispatcherThread.start();
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
log.info("starting event dispatch");
// stream the events from the source buffer without the EOW
// comm channels between reader and writer
Pipe pipe = Pipe.open();
Pipe.SinkChannel writerStream = pipe.sink();
Pipe.SourceChannel readerStream = pipe.source();
writerStream.configureBlocking(true);
readerStream.configureBlocking(false);
// Event writer - Relay in the real world
Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
// Event readers - Clients in the real world
// Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
tReader.setDaemon(true);
tReader.start();
try {
log.info("send first window -- that one should be OK");
StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
Assert.assertEquals(numEventsPerWindow + 1, streamRes.getNumEventsStreamed());
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 1 == callbackStats.getNumSysEventsProcessed();
}
}, "first window processed", 5000, log);
log.info("send the second partial window -- that one should cause an error");
streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win2Size));
Assert.assertEquals(numOfFailureEvent - numEventsPerWindow, streamRes.getNumEventsStreamed());
log.info("wait for dispatcher to finish");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.info("events received: " + callbackStats.getNumDataEventsReceived());
return numOfFailureEvent <= callbackStats.getNumDataEventsProcessed();
}
}, "all events until the error processed", 5000, log);
log.info("all data events have been received but no EOW");
Assert.assertEquals(numOfFailureEvent, clientStats.getTotalStats().getNumDataEvents());
Assert.assertEquals(1, clientStats.getTotalStats().getNumSysEvents());
// at least one failing event therefore < numOfFailureEvent events can be processed
Assert.assertTrue(numOfFailureEvent <= callbackStats.getNumDataEventsProcessed());
// onDataEvent callbacks for e2_1 and e2_2 get cancelled
Assert.assertEquals(2, callbackStats.getNumDataErrorsProcessed());
// only one EOW
Assert.assertEquals(1, callbackStats.getNumSysEventsProcessed());
log.info("Send the remainder of the window");
streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(100000));
// remaining events + EOWs
Assert.assertEquals(srcTestEvents.size() + numWindows - (numOfFailureEvent + 1), streamRes.getNumEventsStreamed());
log.info("wait for the rollback");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 1 == mockConsumer.getNumRollbacks();
}
}, "rollback seen", 5000, log);
log.info("wait for dispatcher to finish after the rollback");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.info("num windows processed: " + callbackStats.getNumSysEventsProcessed());
return numWindows == callbackStats.getNumSysEventsProcessed();
}
}, "all events processed", 5000, log);
} finally {
reader.stop();
dispatcher.shutdown();
log.info("all events processed");
verifyNoLocks(null, srcEventsBuf);
verifyNoLocks(null, destEventsBuf);
}
Logger.getLogger("com.linkedin.databus.client").setLevel(saveLevel);
log.info("end\n");
}
use of com.linkedin.databus.core.StreamEventsResult in project databus by linkedin.
the class TestGenericDispatcher method testShutdownBeforeRollback.
@Test(groups = { "small", "functional" })
public /**
* 1. Dispatcher is dispatching 2 window of events.
* 2. First window consumption is successfully done.
* 3. The second window's onStartDataEventSequence() of the callback registered is blocked (interruptible),
* causing the dispatcher to wait.
* 4. At this instant the dispatcher is shut down. The callback is made to return Failure status which would
* cause rollback in normal scenario.
* 5. As the shutdown message is passed, the blocked callback is expected to be interrupted and no rollback
* calls MUST be made.
*/
void testShutdownBeforeRollback() throws Exception {
final Logger log = Logger.getLogger("TestGenericDispatcher.testShutdownBeforeRollback");
log.setLevel(Level.INFO);
// log.getRoot().setLevel(Level.DEBUG);
LOG.info("start");
// generate events
Vector<Short> srcIdList = new Vector<Short>();
srcIdList.add((short) 1);
DbusEventGenerator evGen = new DbusEventGenerator(0, srcIdList);
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
final int numEvents = 8;
final int numEventsPerWindow = 4;
final int payloadSize = 200;
Assert.assertTrue(evGen.generateEvents(numEvents, numEventsPerWindow, 500, payloadSize, srcTestEvents) > 0);
// find out how much data we need to stream for the failure
// account for the EOW event which is < payload size
int win1Size = payloadSize - 1;
for (DbusEvent e : srcTestEvents) {
win1Size += e.size();
}
// serialize the events to a buffer so they can be sent to the client
final TestGenericDispatcherEventBuffer srcEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, srcEventsBuf, null, true);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
// Create destination (client) buffer
final TestGenericDispatcherEventBuffer destEventsBuf = new TestGenericDispatcherEventBuffer(_generic100KBufferStaticConfig);
/**
* Consumer with ability to wait for latch during onStartDataEventSequence()
*/
class TimeoutDESConsumer extends TimeoutTestConsumer {
private final CountDownLatch latch = new CountDownLatch(1);
private int _countStartWindow = 0;
private final int _failedRequestNumber;
public TimeoutDESConsumer(int failedRequestNumber) {
super(1, 1, 0, 0, 0, 0);
_failedRequestNumber = failedRequestNumber;
}
public CountDownLatch getLatch() {
return latch;
}
@Override
public ConsumerCallbackResult onStartDataEventSequence(SCN startScn) {
_countStartWindow++;
if (_countStartWindow == _failedRequestNumber) {
// Wait for the latch to open
try {
latch.await();
} catch (InterruptedException e) {
}
return ConsumerCallbackResult.ERROR_FATAL;
}
return super.onStartDataEventSequence(startScn);
}
@Override
public ConsumerCallbackResult onDataEvent(DbusEvent e, DbusEventDecoder eventDecoder) {
return ConsumerCallbackResult.SUCCESS;
}
public int getNumBeginWindowCalls() {
return _countStartWindow;
}
}
// Create dispatcher
// fail on second window
final TimeoutDESConsumer mockConsumer = new TimeoutDESConsumer(2);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer((DatabusStreamConsumer) mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
final ConsumerCallbackStats callbackStats = new ConsumerCallbackStats(0, "test", "test", true, false, null);
final UnifiedClientStats unifiedStats = new UnifiedClientStats(0, "test", "test.unified");
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newFixedThreadPool(2), // 100 ms budget
100, new StreamConsumerCallbackFactory(callbackStats, unifiedStats), callbackStats, unifiedStats, null, null);
callback.setSourceMap(sourcesMap);
List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sources);
final RelayDispatcher dispatcher = new RelayDispatcher("dispatcher", _genericRelayConnStaticConfig, subs, new InMemoryPersistenceProvider(), destEventsBuf, callback, null, null, null, null, null);
final Thread dispatcherThread = new Thread(dispatcher);
log.info("starting dispatcher thread");
dispatcherThread.start();
// Generate RegisterRespone for schema
HashMap<Long, List<RegisterResponseEntry>> schemaMap = new HashMap<Long, List<RegisterResponseEntry>>();
List<RegisterResponseEntry> l1 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l2 = new ArrayList<RegisterResponseEntry>();
List<RegisterResponseEntry> l3 = new ArrayList<RegisterResponseEntry>();
l1.add(new RegisterResponseEntry(1L, (short) 1, SOURCE1_SCHEMA_STR));
l2.add(new RegisterResponseEntry(2L, (short) 1, SOURCE2_SCHEMA_STR));
l3.add(new RegisterResponseEntry(3L, (short) 1, SOURCE3_SCHEMA_STR));
schemaMap.put(1L, l1);
schemaMap.put(2L, l2);
schemaMap.put(3L, l3);
// Enqueue Necessary messages before starting dispatch
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(sourcesMap.values()));
dispatcher.enqueueMessage(SourcesMessage.createSetSourcesSchemasMessage(schemaMap));
log.info("starting event dispatch");
// comm channels between reader and writer
Pipe pipe = Pipe.open();
Pipe.SinkChannel writerStream = pipe.sink();
Pipe.SourceChannel readerStream = pipe.source();
writerStream.configureBlocking(true);
/*
* Needed for DbusEventBuffer.readEvents() to exit their loops when no more data is available.
* With Pipe mimicking ChunkedBodyReadableByteChannel, we need to make Pipe non-blocking on the
* reader side to achieve the behavior that ChunkedBodyReadableByte channel provides.
*/
readerStream.configureBlocking(false);
// Event writer - Relay in the real world
Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
// Event readers - Clients in the real world
// Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader, "Reader");
tReader.setDaemon(true);
tReader.start();
try {
log.info("send both windows");
StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
// EOP events, presumably?
Assert.assertEquals(// EOP events, presumably?
"num events streamed should equal total number of events plus 2", numEvents + 2, streamRes.getNumEventsStreamed());
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 2 == mockConsumer.getNumBeginWindowCalls();
}
}, "second window processing started", 5000, log);
dispatcher.shutdown();
// remove the barrier
mockConsumer.getLatch().countDown();
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return !dispatcherThread.isAlive();
}
}, "Ensure Dispatcher thread is shutdown", 5000, log);
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return 0 == mockConsumer.getNumRollbacks();
}
}, "Ensure No Rollback is called", 10, log);
} finally {
reader.stop();
}
log.info("end\n");
}
use of com.linkedin.databus.core.StreamEventsResult in project databus by linkedin.
the class ReadEventsRequestProcessor method process.
@Override
public DatabusRequest process(DatabusRequest request) throws IOException, RequestProcessingException, DatabusException {
boolean isDebug = LOG.isDebugEnabled();
try {
ObjectMapper objMapper = new ObjectMapper();
String checkpointString = request.getParams().getProperty(CHECKPOINT_PARAM, null);
String checkpointStringMult = request.getParams().getProperty(CHECKPOINT_PARAM_MULT, null);
int fetchSize = request.getRequiredIntParam(FETCH_SIZE_PARAM);
String formatStr = request.getRequiredStringParam(OUTPUT_FORMAT_PARAM);
Encoding enc = Encoding.valueOf(formatStr.toUpperCase());
String sourcesListStr = request.getParams().getProperty(SOURCES_PARAM, null);
String subsStr = request.getParams().getProperty(SUBS_PARAM, null);
String partitionInfoStr = request.getParams().getProperty(PARTITION_INFO_STRING);
String streamFromLatestSCNStr = request.getParams().getProperty(STREAM_FROM_LATEST_SCN);
String clientMaxEventVersionStr = request.getParams().getProperty(DatabusHttpHeaders.MAX_EVENT_VERSION);
int clientEventVersion = (clientMaxEventVersionStr != null) ? Integer.parseInt(clientMaxEventVersionStr) : DbusEventFactory.DBUS_EVENT_V1;
if (clientEventVersion < 0 || clientEventVersion == 1 || clientEventVersion > DbusEventFactory.DBUS_EVENT_V2) {
throw new InvalidRequestParamValueException(COMMAND_NAME, DatabusHttpHeaders.MAX_EVENT_VERSION, clientMaxEventVersionStr);
}
if (null == sourcesListStr && null == subsStr) {
throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM + "|" + SUBS_PARAM, "null");
}
// TODO for now we separte the code paths to limit the impact on existing Databus 2 deployments (DDSDBUS-79)
// We have to get rid of this eventually and have a single data path.
boolean v2Mode = null == subsStr;
DbusKeyCompositeFilter keyCompositeFilter = null;
if (null != partitionInfoStr) {
try {
Map<Long, DbusKeyFilter> fMap = KeyFilterConfigJSONFactory.parseSrcIdFilterConfigMap(partitionInfoStr);
keyCompositeFilter = new DbusKeyCompositeFilter();
keyCompositeFilter.setFilterMap(fMap);
if (isDebug)
LOG.debug("keyCompositeFilter is :" + keyCompositeFilter);
} catch (Exception ex) {
String msg = "Got exception while parsing partition Configs. PartitionInfo is:" + partitionInfoStr;
LOG.error(msg, ex);
throw new InvalidRequestParamValueException(COMMAND_NAME, PARTITION_INFO_STRING, partitionInfoStr);
}
}
boolean streamFromLatestSCN = false;
if (null != streamFromLatestSCNStr) {
streamFromLatestSCN = Boolean.valueOf(streamFromLatestSCNStr);
}
long start = System.currentTimeMillis();
List<DatabusSubscription> subs = null;
// parse source ids
SourceIdNameRegistry srcRegistry = _relay.getSourcesIdNameRegistry();
HashSet<Integer> sourceIds = new HashSet<Integer>();
if (null != sourcesListStr) {
String[] sourcesList = sourcesListStr.split(",");
for (String sourceId : sourcesList) {
try {
Integer srcId = Integer.valueOf(sourceId);
sourceIds.add(srcId);
} catch (NumberFormatException nfe) {
HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
if (null != globalHttpStatsCollector) {
globalHttpStatsCollector.registerInvalidStreamRequest();
}
throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, sourceId);
}
}
}
// process explicit subscriptions and generate respective logical partition filters
NavigableSet<PhysicalPartitionKey> ppartKeys = null;
if (null != subsStr) {
List<DatabusSubscription.Builder> subsBuilder = null;
subsBuilder = objMapper.readValue(subsStr, new TypeReference<List<DatabusSubscription.Builder>>() {
});
subs = new ArrayList<DatabusSubscription>(subsBuilder.size());
for (DatabusSubscription.Builder subBuilder : subsBuilder) {
subs.add(subBuilder.build());
}
ppartKeys = new TreeSet<PhysicalPartitionKey>();
for (DatabusSubscription sub : subs) {
PhysicalPartition ppart = sub.getPhysicalPartition();
if (ppart.isAnyPartitionWildcard()) {
ppartKeys = _eventBuffer.getAllPhysicalPartitionKeys();
break;
} else {
ppartKeys.add(new PhysicalPartitionKey(ppart));
}
}
}
// Need to make sure that we don't have tests that send requests in this form.
if (subs != null && checkpointStringMult == null && checkpointString != null) {
throw new RequestProcessingException("Both Subscriptions and CheckpointMult should be present");
}
// convert source ids into subscriptions
if (null == subs)
subs = new ArrayList<DatabusSubscription>();
for (Integer srcId : sourceIds) {
LogicalSource lsource = srcRegistry.getSource(srcId);
if (lsource == null)
throw new InvalidRequestParamValueException(COMMAND_NAME, SOURCES_PARAM, srcId.toString());
if (isDebug)
LOG.debug("registry returns " + lsource + " for srcid=" + srcId);
DatabusSubscription newSub = DatabusSubscription.createSimpleSourceSubscription(lsource);
subs.add(newSub);
}
DbusFilter ppartFilters = null;
if (subs.size() > 0) {
try {
ppartFilters = _eventBuffer.constructFilters(subs);
} catch (DatabusException de) {
throw new RequestProcessingException("unable to generate physical partitions filters:" + de.getMessage(), de);
}
}
ConjunctionDbusFilter filters = new ConjunctionDbusFilter();
// Source filter comes first
if (v2Mode)
filters.addFilter(new SourceDbusFilter(sourceIds));
else if (null != ppartFilters)
filters.addFilter(ppartFilters);
/*
// Key range filter comes next
if ((keyMin >0) && (keyMax > 0))
{
filters.addFilter(new KeyRangeFilter(keyMin, keyMax));
}
*/
if (null != keyCompositeFilter) {
filters.addFilter(keyCompositeFilter);
}
// need to update registerStreamRequest to support Mult checkpoint TODO (DDSDBUS-80)
// temp solution
// 3 options:
// 1. checkpointStringMult not null - generate checkpoint from it
// 2. checkpointStringMult null, checkpointString not null - create empty CheckpointMult
// and add create Checkpoint(checkpointString) and add it to cpMult;
// 3 both are null - create empty CheckpointMult and add empty Checkpoint to it for each ppartition
PhysicalPartition pPartition;
Checkpoint cp = null;
CheckpointMult cpMult = null;
if (checkpointStringMult != null) {
try {
cpMult = new CheckpointMult(checkpointStringMult);
} catch (InvalidParameterSpecException e) {
LOG.error("Invalid CheckpointMult:" + checkpointStringMult, e);
throw new InvalidRequestParamValueException("stream", "CheckpointMult", checkpointStringMult);
}
} else {
// there is no checkpoint - create an empty one
cpMult = new CheckpointMult();
Iterator<Integer> it = sourceIds.iterator();
while (it.hasNext()) {
Integer srcId = it.next();
pPartition = _eventBuffer.getPhysicalPartition(srcId);
if (pPartition == null)
throw new RequestProcessingException("unable to find physical partitions for source:" + srcId);
if (checkpointString != null) {
cp = new Checkpoint(checkpointString);
} else {
cp = new Checkpoint();
cp.setFlexible();
}
cpMult.addCheckpoint(pPartition, cp);
}
}
if (isDebug)
LOG.debug("checkpointStringMult = " + checkpointStringMult + ";singlecheckpointString=" + checkpointString + ";CPM=" + cpMult);
// of the server context.
if (cpMult.getCursorPartition() == null) {
cpMult.setCursorPartition(request.getCursorPartition());
}
if (isDebug) {
if (cpMult.getCursorPartition() != null) {
LOG.debug("Using physical paritition cursor " + cpMult.getCursorPartition());
}
}
// for registerStreamRequest we need a single Checkpoint (TODO - fix it) (DDSDBUS-81)
if (cp == null) {
Iterator<Integer> it = sourceIds.iterator();
if (it.hasNext()) {
Integer srcId = it.next();
pPartition = _eventBuffer.getPhysicalPartition(srcId);
cp = cpMult.getCheckpoint(pPartition);
} else {
cp = new Checkpoint();
cp.setFlexible();
}
}
if (null != checkpointString && isDebug)
LOG.debug("About to stream from cp: " + checkpointString.toString());
HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
HttpStatisticsCollector connHttpStatsCollector = null;
if (null != globalHttpStatsCollector) {
connHttpStatsCollector = (HttpStatisticsCollector) request.getParams().get(globalHttpStatsCollector.getName());
}
if (null != globalHttpStatsCollector)
globalHttpStatsCollector.registerStreamRequest(cp, sourceIds);
StatsCollectors<DbusEventsStatisticsCollector> statsCollectors = _relay.getOutBoundStatsCollectors();
try {
DbusEventBufferBatchReadable bufRead = v2Mode ? _eventBuffer.getDbusEventBufferBatchReadable(sourceIds, cpMult, statsCollectors) : _eventBuffer.getDbusEventBufferBatchReadable(cpMult, ppartKeys, statsCollectors);
int eventsRead = 0;
int minPendingEventSize = 0;
StreamEventsResult result = null;
bufRead.setClientMaxEventVersion(clientEventVersion);
if (v2Mode) {
result = bufRead.streamEvents(streamFromLatestSCN, fetchSize, request.getResponseContent(), enc, filters);
eventsRead = result.getNumEventsStreamed();
minPendingEventSize = result.getSizeOfPendingEvent();
if (isDebug) {
LOG.debug("Process: streamed " + eventsRead + " from sources " + Arrays.toString(sourceIds.toArray()));
// can be used for debugging to stream from a cp
LOG.debug("CP=" + cpMult);
}
// if (null != statsCollectors) statsCollectors.mergeStatsCollectors();
} else {
result = bufRead.streamEvents(streamFromLatestSCN, fetchSize, request.getResponseContent(), enc, filters);
eventsRead = result.getNumEventsStreamed();
minPendingEventSize = result.getSizeOfPendingEvent();
if (isDebug)
LOG.debug("Process: streamed " + eventsRead + " with subscriptions " + subs);
cpMult = bufRead.getCheckpointMult();
if (cpMult != null) {
request.setCursorPartition(cpMult.getCursorPartition());
}
}
if (eventsRead == 0 && minPendingEventSize > 0) {
// Append a header to indicate to the client that we do have at least one event to
// send, but it is too large to fit into client's offered buffer.
request.getResponseContent().addMetadata(DatabusHttpHeaders.DATABUS_PENDING_EVENT_SIZE, minPendingEventSize);
LOG.debug("Returning 0 events but have pending event of size " + minPendingEventSize);
}
} catch (ScnNotFoundException snfe) {
if (null != globalHttpStatsCollector) {
globalHttpStatsCollector.registerScnNotFoundStreamResponse();
}
throw new RequestProcessingException(snfe);
} catch (OffsetNotFoundException snfe) {
LOG.error("OffsetNotFound", snfe);
if (null != globalHttpStatsCollector) {
globalHttpStatsCollector.registerScnNotFoundStreamResponse();
}
throw new RequestProcessingException(snfe);
}
if (null != connHttpStatsCollector) {
connHttpStatsCollector.registerStreamResponse(System.currentTimeMillis() - start);
globalHttpStatsCollector.merge(connHttpStatsCollector);
connHttpStatsCollector.reset();
} else if (null != globalHttpStatsCollector) {
globalHttpStatsCollector.registerStreamResponse(System.currentTimeMillis() - start);
}
} catch (InvalidRequestParamValueException e) {
HttpStatisticsCollector globalHttpStatsCollector = _relay.getHttpStatisticsCollector();
if (null != globalHttpStatsCollector) {
globalHttpStatsCollector.registerInvalidStreamRequest();
}
throw e;
}
return request;
}
Aggregations