use of com.linkedin.databus.client.DatabusSourcesConnection in project databus by linkedin.
the class ClientStateRequestProcessor method processRegistrationInfo.
/**
* Provides an individual registrations details. The individual registration can be
* either that of V2/V3 and top-level or child. Top-level registrations are those that
* were created as a result of one of "registerXXX()" calls on databus-client. In the
* case of multi-partition registrations (like MPRegistration, V2/V3 CLB), only the
* parent registration is considered the top-level registration. Per-partition (child)
* registrations which were created as part of partition migration are NOT top-level
* registrations. The output format can be different depending on whether it is a V2/V3
* as we are dumping the entire Registration in the case of V2. In the case of V3, we
* create an intermediate objects. These are legacy formats which when changed could
* cause the integ-tests to fail.
*
* @param request
* DatabusRequest corresponding to the REST call.
* @throws IOException
* if unable to write to output channel.
* @throws RequestProcessingException
* when registration could not be located.
*/
private void processRegistrationInfo(DatabusRequest request) throws IOException, RequestProcessingException {
boolean found = true;
// V2 Registration lookup first
RegistrationStatsInfo regStatsInfo = null;
try {
DatabusRegistration r = findV2Registration(request, REGISTRATION_KEY_PREFIX);
writeJsonObjectToResponse(r, request);
} catch (RequestProcessingException ex) {
found = false;
}
// V3 Registration lookup if not found
if (!found) {
// if
DatabusV3Registration reg = findV3Registration(request, REGISTRATION_KEY_PREFIX);
// reg
// is
// null,
// the
// callee
// throws
// an
// exception.
DatabusSourcesConnection sourcesConn = _client.getDatabusSourcesConnection(reg.getRegistrationId().getId());
regStatsInfo = new RegistrationStatsInfo(reg, sourcesConn);
writeJsonObjectToResponse(regStatsInfo, request);
}
}
use of com.linkedin.databus.client.DatabusSourcesConnection in project databus by linkedin.
the class TestDatabusRelayMain method testRelayChainingSlowConsumer.
@Test
public /**
* Slow consumer; results in getting SCN not found exception ; as chained relay overwrites buffer .
* Makes sure chained relay overwrites buffer;just like main buffer
*/
void testRelayChainingSlowConsumer() {
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 10;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = 11993;
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1012, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
// create chained relay with only 1 MB buffer
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1;
DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1013, chainedRelayPort, 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay2);
resetSCN(relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
// now create client:
String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
String serverName = "localhost:" + chainedRelayPort;
// create slow consumer with 100ms delay
CountingConsumer countingConsumer = new CountingConsumer(500);
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 1000, 15 * 1000, 1, true);
cr = new ClientRunner(clientConn);
// async starts for all components;
r1.start();
// start chained relay
r2.start();
// start slow client
cr.start();
// let it run for 20 seconds
Thread.sleep(20 * 1000);
r1.pause();
// wait until client got all events or for maxTimeout;
long maxTimeOutMs = 100 * 1000;
long startTime = System.currentTimeMillis();
DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
Assert.assertTrue(stats.getNumSysEvents() > 0);
while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
Thread.sleep(500);
if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
break;
}
}
DbusEventsTotalStats statsChained = relay2.getInboundEventStatisticsCollector().getTotalStats();
LOG.info("Client stats=" + countingConsumer);
LOG.info("Event windows generated=" + stats.getNumSysEvents());
LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
Assert.assertTrue(stats.getMinScn() < statsChained.getMinScn());
Assert.assertTrue(stats.getNumDataEvents() == statsChained.getNumDataEvents());
Assert.assertTrue(stats.getNumSysEvents() == statsChained.getNumSysEvents());
Assert.assertTrue(countingConsumer.getNumErrors() > 0);
Assert.assertTrue(stats.getNumDataEvents() > countingConsumer.getNumDataEvents());
Assert.assertTrue(countingConsumer.getNumSources() == 2);
Assert.assertTrue(stats.getNumSysEvents() > countingConsumer.getNumWindows());
} catch (Exception e) {
LOG.error("Exception: " + e);
Assert.assertTrue(false);
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
}
}
use of com.linkedin.databus.client.DatabusSourcesConnection in project databus by linkedin.
the class TestDatabusRelayMain method testRelayChainingConnectFailure.
@Test
public /** Client and chained relay start before relay, simulating unavailable relay and retries at chained relay and client
*
*/
void testRelayChainingConnectFailure() {
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.fake.FakeSchema", "com.linkedin.events.example.person.Person" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 10;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = 11993;
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1004, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
// create chained relay
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1;
DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1005, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
resetSCN(relay2);
// now create client:
String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
String serverName = "localhost:" + chainedRelayPort;
CountingConsumer countingConsumer = new CountingConsumer();
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
cr = new ClientRunner(clientConn);
// start chained relay
r2.start();
// start client
cr.start();
// Let them try for 5seconds
Thread.sleep(5 * 1000);
r1.start();
// Let the relay run for 10s
Thread.sleep(10 * 1000);
r1.pause();
// wait until client got all events or for maxTimeout;
long maxTimeOutMs = 5 * 1000;
long startTime = System.currentTimeMillis();
DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
Thread.sleep(500);
// stats.getSizeDataEvents());
if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
break;
}
}
LOG.info("Client stats=" + countingConsumer);
LOG.info("Event windows generated=" + stats.getNumSysEvents());
LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
Assert.assertEquals(stats.getNumDataEvents(), countingConsumer.getNumDataEvents());
Assert.assertEquals(countingConsumer.getNumSources(), 2);
Assert.assertEquals(stats.getNumSysEvents(), countingConsumer.getNumWindows());
} catch (Exception e) {
LOG.error("Exception: " + e);
Assert.assertTrue(false);
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
}
}
use of com.linkedin.databus.client.DatabusSourcesConnection in project databus by linkedin.
the class TestDatabusRelayMain method testRelayChainingPauseResume.
@Test
public /**
* Regular concurrent consumption amongst db-relay , chained relay and client. The only twist is that the chained relay
* will be paused and resumed
*/
void testRelayChainingPauseResume() {
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 10;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = 11993;
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1014, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
// create chained relay
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1;
DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1015, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay2);
resetSCN(relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
// now create client:
String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
String serverName = "localhost:" + chainedRelayPort;
CountingConsumer countingConsumer = new CountingConsumer();
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
cr = new ClientRunner(clientConn);
// async starts for all components;
r1.start();
// start chained relay
r2.start();
// start client
cr.start();
// let them run for 2 seconds
Thread.sleep(2 * 1000);
// pause chained relay
r2.pause();
// Sleep again for some time
Thread.sleep(2 * 1000);
// stop relay
r1.pause();
// resume chained relay
r2.unpause();
// wait until client got all events or for maxTimeout;
long maxTimeOutMs = 5 * 1000;
long startTime = System.currentTimeMillis();
DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
Thread.sleep(500);
// stats.getSizeDataEvents());
if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
break;
}
}
LOG.info("Client stats=" + countingConsumer);
LOG.info("Event windows generated=" + stats.getNumSysEvents());
LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
Assert.assertTrue(stats.getNumDataEvents() == countingConsumer.getNumDataEvents());
Assert.assertTrue(countingConsumer.getNumSources() == 2);
Assert.assertTrue(stats.getNumSysEvents() == countingConsumer.getNumWindows());
} catch (Exception e) {
LOG.error("Exception: " + e);
Assert.assertTrue(false);
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
}
}
use of com.linkedin.databus.client.DatabusSourcesConnection in project databus by linkedin.
the class TestDatabusRelayMain method testDynamicBufferGrowthClient.
@Test
public /**
* Basic consumption client and by chained relay from regular relay
* but start with small readBuffer (20K) - Set up large events
*/
void testDynamicBufferGrowthClient() throws InterruptedException, InvalidConfigException {
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
final Logger log = Logger.getLogger("TestDatabusRelayMain.testDynamicBufferGrowth");
Logger.getRootLogger().setLevel(Level.INFO);
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 20;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 100, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = Utils.getAvailablePort(11994);
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1002, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertNotNull(relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
log.info("now create client");
String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
String serverName = "localhost:" + relayPort;
final CountingConsumer countingConsumer = new CountingConsumer();
//Set maxSize to 100K, maxEventSize to 15k, set init readBufferSize to maxEventSize/2
int maxEventSize = 15 * 1024;
int initReadBufferSize = maxEventSize / 2;
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 100 * 1024, maxEventSize, 30 * 1000, 100, 30 * 1000, 1, true, initReadBufferSize);
cr = new ClientRunner(clientConn);
log.info("async starts for all components");
r1.start();
assertRelayRunning(r1.getRelay(), 5000, log);
Thread.sleep(5 * 1000);
r1.pause();
final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
log.info("start the client");
cr.start();
// wait until client got all events or for maxTimeout;
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
log.debug("countingConsumer.getNumWindows()=" + countingConsumer.getNumWindows());
return countingConsumer.getNumWindows() == stats.getNumSysEvents();
}
}, "wait until client got all events", 10000, log);
LOG.info("Client stats=" + countingConsumer);
LOG.info("Event windows generated=" + stats.getNumSysEvents());
LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
Assert.assertEquals(stats.getNumDataEvents(), countingConsumer.getNumDataEvents());
Assert.assertEquals(countingConsumer.getNumSources(), 2);
Assert.assertEquals(stats.getNumSysEvents(), countingConsumer.getNumWindows());
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
}
}
Aggregations