use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.
the class TestRelayCommandsLocal method testSources1Command.
@Test
public void testSources1Command() throws Exception {
LOG.debug("\n\nstarting testSources1Command()\n");
HttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/sources");
SimpleTestHttpClient httpClient = SimpleTestHttpClient.createLocal(TimeoutPolicy.ALL_TIMEOUTS);
SimpleHttpResponseHandler respHandler = httpClient.sendRequest(_serverAddress, httpRequest);
assertTrue("failed to get a response", respHandler.awaitResponseUninterruptedly(1, TimeUnit.SECONDS));
ByteArrayInputStream in = new ByteArrayInputStream(respHandler.getReceivedBytes());
ObjectMapper objMapper = new ObjectMapper();
List<IdNamePair> res = objMapper.readValue(in, new TypeReference<List<IdNamePair>>() {
});
assertNotNull("no result", res);
if (LOG.isDebugEnabled()) {
LOG.debug("/sources response:" + new String(respHandler.getReceivedBytes()));
}
HashSet<IdNamePair> origSet = new HashSet<IdNamePair>(_staticConfig.getSourceIds());
HashSet<IdNamePair> resSet = new HashSet<IdNamePair>(res);
Assert.assertEquals(origSet, resSet);
}
use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.
the class GenericDispatcher method doCheckStartSource.
protected boolean doCheckStartSource(DispatcherState curState, Long eventSrcId, SchemaId schemaId) {
boolean success = true;
if (eventSrcId >= 0) {
IdNamePair source = curState.getSources().get(eventSrcId);
if (null == source) {
_log.error("Unable to find source: srcid=" + eventSrcId);
success = false;
} else {
VersionedSchema verSchema = curState.getSchemaSet().getLatestVersionByName(source.getName());
VersionedSchema exactSchema = _schemaIdCheck ? curState.getSchemaSet().getById(schemaId) : null;
if (null == verSchema) {
_log.error("Unable to find schema: srcid=" + source.getId() + " name=" + source.getName());
success = false;
} else if (_schemaIdCheck && null == exactSchema) {
_log.error("Unable to find schema: srcid=" + source.getId() + " name=" + source.getName() + " schemaId=" + schemaId);
success = false;
} else if (verSchema.getSchema() != curState.getCurrentSourceSchema()) {
curState.switchToStartStreamSource(source, verSchema.getSchema());
success = doStartStreamSource(curState);
}
}
}
return success;
}
use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.
the class RelayPullThread method doSourcesResponseSuccess.
protected void doSourcesResponseSuccess(ConnectionState curState) {
mergeRelayCallsStats();
Map<String, IdNamePair> sourceNameMap = curState.getSourcesNameMap();
StringBuilder sb = new StringBuilder();
boolean firstSource = true;
boolean error = false;
List<IdNamePair> sourcesList = new ArrayList<IdNamePair>(_sourcesConn.getSourcesNames().size());
if (curState.getRelayConnection().getProtocolVersion() < 3) {
// we pass a list of subscriptions.
for (String sourceName : _sourcesConn.getSourcesNames()) {
IdNamePair source = sourceNameMap.get(sourceName);
if (null == source) {
_log.error("Source not found on server: " + sourceName);
error = true;
break;
} else {
if (!firstSource)
sb.append(',');
sb.append(source.getId().toString());
firstSource = false;
sourcesList.add(source);
}
}
}
String sourcesIdList = sb.toString();
String subsString = error ? "ERROR" : buildSubsList(curState.getSubscriptions(), sourceNameMap);
if (_log.isDebugEnabled()) {
_log.debug("Source ids: " + sourcesIdList);
_log.debug("Subs : " + subsString);
}
_sourcesConn.getRelayDispatcher().enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(curState.getSources(), sourcesIdList));
if (toTearConnAfterHandlingResponse()) {
tearConnectionAndEnqueuePickServer();
} else {
if (error) {
curState.switchToPickServer();
} else {
curState.switchToRequestSourcesSchemas(sourcesIdList, subsString);
if (_sourcesConn.isBootstrapEnabled()) {
_sourcesConn.getBootstrapPuller().enqueueMessage(SourcesMessage.createSetSourcesIdsMessage(curState.getSources(), curState.getSourcesIdListString()));
}
String hostHdr = curState.getHostName(), svcHdr = curState.getSvcName();
_log.info("Connected to relay " + hostHdr + " with a service identifier " + svcHdr);
}
enqueueMessage(curState);
}
}
use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.
the class RelayPullThread method doRequestStream.
protected void doRequestStream(ConnectionState curState) {
boolean debugEnabled = _log.isDebugEnabled();
if (debugEnabled)
_log.debug("Checking for free space in buffer");
int freeBufferThreshold = (int) (_sourcesConn.getConnectionConfig().getFreeBufferThreshold() * 100.0 / _pullerBufferUtilizationPct);
try {
curState.getDataEventsBuffer().waitForFreeSpace(freeBufferThreshold);
} catch (InterruptedException ie) {
//loop
enqueueMessage(curState);
return;
}
Checkpoint cp = curState.getCheckpoint();
if (debugEnabled)
_log.debug("Checkpoint at RequestDataEvents: " + cp.toString());
if (null == _relayFilter) {
if (debugEnabled)
_log.debug("Initializing relay filter config");
_relayFilter = new DbusKeyCompositeFilter();
Map<String, IdNamePair> srcNameIdMap = curState.getSourcesNameMap();
for (DbusKeyCompositeFilterConfig conf : _relayFilterConfigs) {
Map<String, KeyFilterConfigHolder> cMap = conf.getConfigMap();
Map<Long, KeyFilterConfigHolder> fConfMap = new HashMap<Long, KeyFilterConfigHolder>();
for (Entry<String, KeyFilterConfigHolder> e : cMap.entrySet()) {
IdNamePair idName = srcNameIdMap.get(e.getKey());
if (null != idName) {
fConfMap.put(idName.getId(), e.getValue());
}
}
if (debugEnabled)
_log.debug("FilterConfMap is :" + fConfMap);
_relayFilter.merge(new DbusKeyCompositeFilter(fConfMap));
}
if (debugEnabled)
_log.debug("Merged Filter (before deduping) is :" + _relayFilter);
_relayFilter.dedupe();
if (debugEnabled)
_log.debug("Merged Filter (after deduping) is :" + _relayFilter);
}
_streamCallStartMs = System.currentTimeMillis();
if (null != _relayCallsStats)
_relayCallsStats.registerStreamRequest(cp, EMPTY_STREAM_LIST);
int fetchSize = (int) ((curState.getDataEventsBuffer().getBufferFreeReadSpace() / 100.0) * _pullerBufferUtilizationPct);
fetchSize = Math.max(freeBufferThreshold, fetchSize);
CheckpointMult cpMult = new CheckpointMult();
String args;
if (curState.getRelayConnection().getProtocolVersion() >= 3) {
// for version 3 and higher we pass subscriptions
args = curState.getSubsListString();
for (DatabusSubscription sub : curState.getSubscriptions()) {
PhysicalPartition p = sub.getPhysicalPartition();
cpMult.addCheckpoint(p, cp);
}
} else {
args = curState.getSourcesIdListString();
cpMult.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
}
curState.switchToStreamRequestSent();
sendHeartbeat(_sourcesConn.getUnifiedClientStats());
curState.getRelayConnection().requestStream(args, _relayFilter, fetchSize, cpMult, _sourcesConn.getConnectionConfig().getKeyRange(), curState);
}
use of com.linkedin.databus.core.util.IdNamePair in project databus by linkedin.
the class TestMultiConsumerCallback method test1StreamConsumerHappyPath.
@Test(groups = { "small", "functional" })
public void test1StreamConsumerHappyPath() {
LOG.info("\n\nstarting test1StreamConsumerHappyPath()");
Hashtable<Long, AtomicInteger> keyCounts = new Hashtable<Long, AtomicInteger>();
DbusEventBuffer eventsBuf = new DbusEventBuffer(_generic100KBufferStaticConfig);
eventsBuf.start(0);
eventsBuf.startEvents();
initBufferWithEvents(eventsBuf, 1, 1, (short) 1, keyCounts);
initBufferWithEvents(eventsBuf, 2, 2, (short) 3, keyCounts);
eventsBuf.endEvents(100L);
DatabusStreamConsumer mockConsumer = EasyMock.createStrictMock(DatabusStreamConsumer.class);
SelectingDatabusCombinedConsumer sdccMockConsumer = new SelectingDatabusCombinedConsumer(mockConsumer);
List<String> sources = new ArrayList<String>();
Map<Long, IdNamePair> sourcesMap = new HashMap<Long, IdNamePair>();
for (int i = 1; i <= 3; ++i) {
IdNamePair sourcePair = new IdNamePair((long) i, "source" + i);
sources.add(sourcePair.getName());
sourcesMap.put(sourcePair.getId(), sourcePair);
}
DatabusV2ConsumerRegistration consumerReg = new DatabusV2ConsumerRegistration(sdccMockConsumer, sources, null);
List<DatabusV2ConsumerRegistration> allRegistrations = Arrays.asList(consumerReg);
MultiConsumerCallback callback = new MultiConsumerCallback(allRegistrations, Executors.newCachedThreadPool(), 60000, new StreamConsumerCallbackFactory(null, null), null, null, null, null);
callback.setSourceMap(sourcesMap);
DbusEventBuffer.DbusEventIterator iter = eventsBuf.acquireIterator("myIter1");
assert iter.hasNext() : "unable to read event";
DbusEvent event1 = iter.next();
assert iter.hasNext() : "unable to read event";
DbusEvent event2 = iter.next();
assert iter.hasNext() : "unable to read event";
DbusEvent event3 = iter.next();
initMockStreamConsumer3EventFullLifecycle(mockConsumer, event1, event2, event3, keyCounts);
assert3EventFullLifecycle(callback, event1, event2, event3);
EasyMock.verify(mockConsumer);
assert keyCounts.get(1L).get() == 1 : "invalid number of event(1) calls: " + keyCounts.get(1L).get();
assert keyCounts.get(2L).get() == 1 : "invalid number of event(2) calls:" + keyCounts.get(2L).get();
assert keyCounts.get(3L).get() == 1 : "invalid number of event(3) calls:" + keyCounts.get(3L).get();
}
Aggregations