use of com.linkedin.databus2.relay.DatabusRelayMain in project databus by linkedin.
the class TestDatabusRelayMain method testRelalyChainingPartialSubscribe.
@Test
public /**
* Client consumes subset of sources from chained relay
*/
void testRelalyChainingPartialSubscribe() {
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 10;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = 11993;
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1006, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
// create chained relay
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1;
DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1007, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay2);
resetSCN(relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
// now create client:
String srcSubscriptionString = "com.linkedin.events.example.Settings";
String serverName = "localhost:" + chainedRelayPort;
CountingConsumer countingConsumer = new CountingConsumer();
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, -1, 30 * 1000, 100, 15 * 1000, 1, true);
cr = new ClientRunner(clientConn);
// start relay r1
r1.start();
// start chained relay
r2.start();
// start client
cr.start();
// Let the relay run for 10s
Thread.sleep(10 * 1000);
r1.pause();
// wait until client got all events or for maxTimeout;
long maxTimeOutMs = 5 * 1000;
long startTime = System.currentTimeMillis();
DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
Thread.sleep(500);
// stats.getSizeDataEvents());
if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
break;
}
}
LOG.info("Client stats=" + countingConsumer);
LOG.info("Event windows generated=" + stats.getNumSysEvents());
LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
Assert.assertTrue(countingConsumer.getNumSources() == 1);
Assert.assertTrue(stats.getNumSysEvents() == countingConsumer.getNumWindows());
Assert.assertTrue(stats.getNumDataEvents() == 2 * countingConsumer.getNumDataEvents());
} catch (Exception e) {
LOG.error("Exception: " + e);
Assert.assertTrue(false);
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
}
}
use of com.linkedin.databus2.relay.DatabusRelayMain in project databus by linkedin.
the class TestDatabusRelayMain method testRelayChainingRestartSCNOffset.
@Test
public /**
* Test relay chaining that utilizes restart SCN offset
*/
void testRelayChainingRestartSCNOffset() throws InterruptedException, InvalidConfigException {
final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayChainingRestartSCNOffset");
log.info("start");
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null, r3 = null;
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 2;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = 11993;
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1016, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
// create chained relay
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1;
DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1017, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
resetSCN(relay2);
// now create client:
String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
String serverName = "localhost:" + chainedRelayPort;
final CountingConsumer countingConsumer = new CountingConsumer();
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
cr = new ClientRunner(clientConn);
// async starts for all components;
r1.start();
// start chained relay
r2.start();
// start client
Thread.sleep(500);
cr.start();
r1.pause();
// let it run for 10 seconds
Thread.sleep(10 * 1000);
// wait until client got all events or for maxTimeout;
final long maxTimeOutMs = 5 * 1000;
final DbusEventsTotalStats dbRelayStats = relay1.getInboundEventStatisticsCollector().getTotalStats();
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return countingConsumer.getNumWindows() == dbRelayStats.getNumSysEvents();
}
}, "consumer caught up", maxTimeOutMs, log);
log.info("Client stats=" + countingConsumer);
log.info("Event windows generated=" + dbRelayStats.getNumSysEvents());
log.info("numDataEvents=" + dbRelayStats.getNumDataEvents() + " numWindows=" + dbRelayStats.getNumSysEvents() + " size=" + dbRelayStats.getSizeDataEvents());
Assert.assertEquals(dbRelayStats.getNumDataEvents(), countingConsumer.getNumDataEvents());
Assert.assertEquals(countingConsumer.getNumSources(), 2);
Assert.assertEquals(dbRelayStats.getNumSysEvents(), countingConsumer.getNumWindows());
cr.shutdown();
boolean s2 = r2.shutdown(2000);
Assert.assertTrue(s2);
Assert.assertTrue(!r2.isAlive());
//start r3; new chained relay with restart SCN offset; we get some data;
chainedSrcConfigs[0].setRestartScnOffset(dbRelayStats.getMaxScn() - dbRelayStats.getPrevScn());
DatabusRelayMain relay3 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1018, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
r3 = new DatabusRelayTestUtil.RelayRunner(relay3);
r3.start();
final DbusEventsTotalStats newChainedRlyStats = relay3.getInboundEventStatisticsCollector().getTotalStats();
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return newChainedRlyStats.getNumDataEvents() > 0;
}
}, "new chained relay running", 5000, log);
log.info("Stats3= numDataEvents=" + newChainedRlyStats.getNumDataEvents() + " numWindows=" + newChainedRlyStats.getNumSysEvents() + " size=" + newChainedRlyStats.getSizeDataEvents());
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2, r3 }, cr);
log.info("end");
}
}
use of com.linkedin.databus2.relay.DatabusRelayMain in project databus by linkedin.
the class TestDatabusRelayMain method testClientNoEventsResetConnection.
@Test
public /**
* test resetting connection when there is no events for some period of time
* @throws InterruptedException
* @throws InvalidConfigException
*/
void testClientNoEventsResetConnection() throws InterruptedException, InvalidConfigException {
LOG.setLevel(Level.ALL);
DatabusRelayTestUtil.RelayRunner r1 = null;
ClientRunner cr = null;
try {
String srcName = "com.linkedin.events.example.Settings";
// create main relay with random generator
int eventRatePerSec = 10;
PhysicalSourceConfig srcConfig = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) 1, DatabusRelayTestUtil.getPhysicalSrcName(srcName), "mock", 500, eventRatePerSec, new String[] { srcName });
int relayPort = 11995;
DatabusRelayMain relay = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1001, relayPort, 10 * 1024 * 1024, new PhysicalSourceConfig[] { srcConfig }, SCHEMA_REGISTRY_DIR);
Assert.assertNotEquals(relay, null);
r1 = new DatabusRelayTestUtil.RelayRunner(relay);
// async starts
r1.start();
DbusEventsTotalStats stats = relay.getInboundEventStatisticsCollector().getTotalStats();
// start client in parallel
String srcSubscriptionString = srcName;
String serverName = "localhost:" + relayPort;
ResetsCountingConsumer countingConsumer = new ResetsCountingConsumer();
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
cr = new ClientRunner(clientConn);
cr.start();
// generate some events
TestUtil.sleep(1000);
// pause event generator
// and wait untill all the events are consumed
// but since it is less then timeout the connection should NOT reset
LOG.info("Sending pause to relay!");
r1.pause();
TestUtil.sleep(4000);
LOG.info("no events, time less then threshold. Events=" + countingConsumer.getNumDataEvents() + "; resets = " + countingConsumer.getNumResets());
Assert.assertEquals(countingConsumer.getNumResets(), 0);
// generate more events, more time elapsed then the threshold, but since there are
// events - NO reset
r1.unpause();
Thread.sleep(8000);
LOG.info("some events, more time then timeout. Events=" + countingConsumer.getNumDataEvents() + "; resets = " + countingConsumer.getNumResets());
Assert.assertEquals(countingConsumer.getNumResets(), 0);
// stop events
r1.pause();
//set threshold to 0 completely disabling the feature
clientConn.getRelayPullThread().setNoEventsConnectionResetTimeSec(0);
Thread.sleep(8000);
LOG.info("no events, more time then timeout, but feature disabled. Events=" + countingConsumer.getNumDataEvents() + "; resets = " + countingConsumer.getNumResets());
Assert.assertEquals(countingConsumer.getNumResets(), 0);
// enable the feature, and sleep for timeout
clientConn.getRelayPullThread().setNoEventsConnectionResetTimeSec(5);
// now wait with no events
LOG.info("pause the producer. sleep for 6 sec, should reset");
TestUtil.sleep(6000);
LOG.info("Client stats=" + countingConsumer);
LOG.info("Num resets=" + countingConsumer.getNumResets());
LOG.info("Event windows generated=" + stats.getNumSysEvents());
Assert.assertEquals(countingConsumer.getNumResets(), 0, "0 resets");
Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
boolean stopped = r1.shutdown(2000);
Assert.assertTrue(stopped);
LOG.info("Relay r1 stopped");
cr.shutdown();
LOG.info("Client cr stopped");
Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
}
}
use of com.linkedin.databus2.relay.DatabusRelayMain in project databus by linkedin.
the class TestDatabusRelayEvents method testV2Events.
/**
* Stuffs an event buffer with both a v1 and a v2 event, then reads the buffer two ways:
* first accepting only v1 events (verifying conversion of the v2 event to v1); then accepting
* both v1 and v2 events.
*
* Note that the version of the _EOP_ events must match the version of the event factory,
* regardless of the versions of any preceding "real" events. (This matches DbusEventBuffer
* behavior; see the serializeLongKeyEndOfPeriodMarker() call in endEvents() for details.)
*/
@Test
public void testV2Events() throws KeyTypeNotImplementedException, InvalidEventException, IOException, DatabusException {
final Logger log = Logger.getLogger("TestDatabusRelayEvents.testV2Events");
log.setLevel(Level.DEBUG);
String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
short srcId = 2;
short pId = 1;
int relayPort = Utils.getAvailablePort(11993);
// create relay
final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
DatabusRelayTestUtil.RelayRunner r1 = null;
ClientRunner cr = null;
try {
//EventProducer[] producers = relay1.getProducers();
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
log.info("Relay created");
DbusEventBufferMult bufMult = relay1.getEventBuffer();
PhysicalPartition pPartition = new PhysicalPartition((int) pId, pSourceName);
DbusEventBuffer buf = (DbusEventBuffer) bufMult.getDbusEventBufferAppendable(pPartition);
log.info("create some events");
long windowScn = 100L;
ByteBuffer serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V2);
ReadableByteChannel channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
int readEvents = buf.readEvents(channel);
log.info("successfully read in " + readEvents + " events ");
channel.close();
windowScn = 101L;
serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V1);
channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
readEvents = buf.readEvents(channel);
log.info("successfully read in " + readEvents + " events ");
channel.close();
log.info("starting relay on port " + relayPort);
r1.start();
//TestUtil.sleep(10*1000);
// wait until relay comes up
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return relay1.isRunningStatus();
}
}, "Relay hasn't come up completely ", 30000, LOG);
log.info("now create client");
String srcSubscriptionString = TestUtil.join(srcs, ",");
String serverName = "localhost:" + relayPort;
final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
int id = (RngUtils.randomPositiveInt() % 10000) + 1;
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
cr = new ClientRunner(clientConn);
log.info("starting client");
cr.start();
// wait till client gets the event
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
int events = countingConsumer.getNumDataEvents();
LOG.info("client got " + events + " events");
return events == 2;
}
}, "Consumer didn't get 2 events ", 64 * 1024, LOG);
// asserts
Assert.assertEquals(countingConsumer.getNumDataEvents(), 2);
Assert.assertEquals(countingConsumer.getNumWindows(), 2);
Assert.assertEquals(countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 2);
log.info("shutdown first client");
clientConn.stop();
cr.shutdown();
TestUtil.sleep(1000);
cr = null;
log.info("start another client who understands V2");
final EventsCountingConsumer countingConsumer1 = new EventsCountingConsumer();
clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer1, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V2, 0);
cr = new ClientRunner(clientConn);
cr.start();
log.info("wait till client gets the event");
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
int events = countingConsumer1.getNumDataEvents();
LOG.debug("client got " + events + " events");
return events == 2;
}
}, "Consumer didn't get 2 events ", 64 * 1024, LOG);
// asserts
Assert.assertEquals(countingConsumer1.getNumDataEvents(), 2);
Assert.assertEquals(countingConsumer1.getNumWindows(), 2);
Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 1);
Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V2), 1);
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
}
}
use of com.linkedin.databus2.relay.DatabusRelayMain in project databus by linkedin.
the class TestDatabusRelayEvents method createRelay.
/**
* create a test relay with event producer turned off
*/
private DatabusRelayMain createRelay(int relayPort, int pId, String[] srcs) throws IOException, DatabusException {
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcs.length];
String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) pId, pSourceName, "mock", 500, 0, srcs);
srcConfigs[0] = src1;
HttpRelay.Config httpRelayConfig = DatabusRelayTestUtil.createHttpRelayConfig(1002, relayPort, 3024);
// do not produce any events
httpRelayConfig.setStartDbPuller("false");
httpRelayConfig.getSchemaRegistry().getFileSystem().setSchemaDir("TestDatabusRelayEvents_schemas");
final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelay(srcConfigs, httpRelayConfig);
Assert.assertNotNull(relay1);
return relay1;
}
Aggregations