use of com.linkedin.databus.client.DatabusSourcesConnection in project databus by linkedin.
the class RelayEventProducer method createDatabusSourcesConnection.
public static DatabusSourcesConnection createDatabusSourcesConnection(String producerName, int id, String serverName, String subscriptionString, DatabusCombinedConsumer consumer, long internalBufferMaxSize, int largestEventSize, long consumerTimeoutMs, long pollIntervalMs, long connTimeoutMs, int consumerParallelism, boolean blockingBuffer, DatabusClientNettyThreadPools nettyThreadPools, int noEventsTimeoutSec, int maxEventVersion, int initReadBufferSize) throws InvalidConfigException {
// the assumption here is that the list of subscriptions will become the
// list of sources hosted by the relay
Set<ServerInfo> relayServices = createServerInfo(serverName, subscriptionString);
// null bootstrapService
Set<ServerInfo> bootstrapServices = null;
// create subscription objects based on what is required by subscription
String[] subscriptionList = subscriptionString.split(",");
List<DatabusSubscription> subsList = DatabusSubscription.createSubscriptionList(Arrays.asList(subscriptionList));
List<String> sourcesStrList = DatabusSubscription.getStrList(subsList);
LOG.info("The sourcesList is " + sourcesStrList);
// create registration objects with consumers
List<DatabusV2ConsumerRegistration> relayConsumers = createDatabusV2ConsumerRegistration(consumer, sourcesStrList);
List<DatabusV2ConsumerRegistration> bstConsumers = null;
// setup sources connection config
DatabusSourcesConnection.Config confBuilder = new DatabusSourcesConnection.Config();
confBuilder.setId(id);
// consume whatever is in relay
confBuilder.setConsumeCurrent(true);
//this is set to false as the behaviour is to read the latest SCN when SCN is not found, the buffer isn't cleared
//as such , so a possibility of gaps in events arises. What we want ideally is to clear existing buffer and then consume from latest SCN
confBuilder.setReadLatestScnOnError(false);
// 10s max consumer timeout
confBuilder.setConsumerTimeBudgetMs(consumerTimeoutMs);
// poll interval in ms and infinite retry;
confBuilder.getPullerRetries().setMaxRetryNum(-1);
confBuilder.getPullerRetries().setInitSleep(pollIntervalMs);
confBuilder.setConsumerParallelism(consumerParallelism);
confBuilder.getDispatcherRetries().setMaxRetryNum(1);
// internal buffer conf
DbusEventBuffer.Config bufferConf = new DbusEventBuffer.Config();
bufferConf.setMaxSize(internalBufferMaxSize);
bufferConf.setAllocationPolicy("DIRECT_MEMORY");
if (initReadBufferSize > 0) {
bufferConf.setAverageEventSize(initReadBufferSize);
}
bufferConf.setMaxEventSize(largestEventSize);
//client buffer's scn index- not used
bufferConf.setScnIndexSize(64 * 1024);
String queuePolicy = blockingBuffer ? "BLOCK_ON_WRITE" : "OVERWRITE_ON_WRITE";
bufferConf.setQueuePolicy(queuePolicy);
//get appropriate checkpointThresholdPct
double newCkptPct = confBuilder.computeSafeCheckpointThresholdPct(bufferConf);
if (newCkptPct < 5.0 || newCkptPct > 95.0) {
LOG.warn("Not setting required checkpointThresholdPct : " + newCkptPct + "to accommodate largestEventSize= " + largestEventSize + " in buffer of size " + bufferConf.getMaxSize());
if (newCkptPct <= 0.0) {
//unlikely to happen: if it does retain default
newCkptPct = confBuilder.getCheckpointThresholdPct();
}
if (newCkptPct < 5.0) {
newCkptPct = 5.0;
} else if (newCkptPct > 95.0) {
newCkptPct = 95.0;
}
}
LOG.info("Setting checkpointThresholdPct:" + newCkptPct);
confBuilder.setCheckpointThresholdPct(newCkptPct);
confBuilder.setEventBuffer(bufferConf);
confBuilder.setNoEventsConnectionResetTimeSec(noEventsTimeoutSec);
DatabusSourcesConnection.StaticConfig connConfig = confBuilder.build();
// internal buffers of databus client library
DbusEventBuffer buffer = new DbusEventBuffer(connConfig.getEventBuffer());
buffer.start(0);
DbusEventBuffer bootstrapBuffer = null;
// Create threadpools and netty managers
// read - write timeout in ms
long readTimeoutMs = connTimeoutMs;
long writeTimeoutMs = connTimeoutMs;
long bstReadTimeoutMs = connTimeoutMs;
int protocolVersion = 2;
// connection factory
NettyHttpConnectionFactory defaultConnFactory = new NettyHttpConnectionFactory(nettyThreadPools.getBossExecutorService(), nettyThreadPools.getIoExecutorService(), null, nettyThreadPools.getTimer(), writeTimeoutMs, readTimeoutMs, bstReadTimeoutMs, protocolVersion, maxEventVersion, nettyThreadPools.getChannelGroup());
// Create Thread pool for consumer threads
int maxThreadsNum = 1;
int keepAliveMs = 1000;
ThreadPoolExecutor defaultExecutorService = new OrderedMemoryAwareThreadPoolExecutor(maxThreadsNum, 0, 0, keepAliveMs, TimeUnit.MILLISECONDS);
ConsumerCallbackStats relayConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.cons", producerName + ".inbound.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
ConsumerCallbackStats bootstrapConsumerStats = new ConsumerCallbackStats(id, producerName + ".inbound.bs.cons", producerName + ".inbound.bs.cons", true, false, null, ManagementFactory.getPlatformMBeanServer());
UnifiedClientStats unifiedClientStats = new UnifiedClientStats(id, producerName + ".inbound.unified.cons", producerName + ".inbound.unified.cons", true, false, UnifiedClientStats.DEFAULT_DEADNESS_THRESHOLD_MS, null, ManagementFactory.getPlatformMBeanServer());
DatabusRelayConnectionFactory relayConnFactory = defaultConnFactory;
DatabusBootstrapConnectionFactory bootstrapConnFactory = defaultConnFactory;
ConnectionStateFactory connStateFactory = new ConnectionStateFactory(sourcesStrList);
DatabusSourcesConnection conn = new DatabusSourcesConnection(connConfig, subsList, relayServices, bootstrapServices, relayConsumers, bstConsumers, buffer, bootstrapBuffer, defaultExecutorService, // getContainerStatsCollector(),
null, // getInboundEventStatisticsCollector(),
null, // getBootstrapEventsStatsCollector(),
null, // relay callback stats
relayConsumerStats, // bootstrap callback stats
bootstrapConsumerStats, // combined relay/bootstrap callback stats
unifiedClientStats, // getCheckpointPersistenceProvider(),
null, relayConnFactory, bootstrapConnFactory, // getHttpStatsCollector(),
null, // RegistrationId
null, null, new DbusEventV2Factory(), // TODO Get the ref to factory from HttpRelay.
connStateFactory);
return conn;
}
use of com.linkedin.databus.client.DatabusSourcesConnection in project databus by linkedin.
the class TestDatabusRelayMain method testRelayChainingSCNRegress.
@Test
public /**
* Test regression of SCN of chained relay
*/
void testRelayChainingSCNRegress() throws InvalidConfigException, InterruptedException {
final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayChainingSCNRegress");
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null, r3 = null;
ClientRunner cr = null;
//log.setLevel(Level.DEBUG);
log.info("start");
//DbusEventBuffer.LOG.setLevel(Level.DEBUG);
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 10;
int largestEventSize = 512 * 1024;
long largestWindowSize = 1 * 1024 * 1024;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = 11993;
final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1019, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
final DatabusRelayMain relay3 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1020, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertNotNull(relay1);
Assert.assertNotNull(relay3);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
final DbusEventsTotalStats stats3 = relay3.getInboundEventStatisticsCollector().getTotalStats();
// create chained relay
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, 500, eventRatePerSec, 0, largestEventSize, largestWindowSize, srcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1;
final DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1021, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertNotNull(relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
resetSCN(relay2);
// now create client:
String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
String serverName = "localhost:" + chainedRelayPort;
CountingConsumer countingConsumer = new CountingConsumer();
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, largestEventSize, 30 * 1000, 100, 15 * 1000, 1, true, largestEventSize / 10);
cr = new ClientRunner(clientConn);
// async starts for all components;
r1.start();
Thread.sleep(10 * 1000);
// start chained relay
r2.start();
//start client
cr.start();
//Pause r1;
r1.pause();
Thread.sleep(1000);
long firstGenDataEvents = stats.getNumDataEvents();
long firstGenMinScn = stats.getMinScn();
long firstGenWindows = stats.getNumSysEvents();
Assert.assertTrue(stats.getNumSysEvents() > 0);
log.warn("relay1: numDataEvents=" + firstGenDataEvents + " numWindows=" + firstGenWindows + " minScn=" + firstGenMinScn + " maxScn=" + stats.getMaxScn());
Thread.sleep(4 * 1000);
//clear the relay
boolean s = r1.shutdown(2000);
Assert.assertTrue(s);
DbusEventsTotalStats stats2 = relay2.getInboundEventStatisticsCollector().getTotalStats();
long firstGenChainWindows = stats2.getNumSysEvents();
log.warn("relay2: numDataEvents=" + stats2.getNumDataEvents() + " numWindows=" + firstGenChainWindows + " minScn=" + stats2.getMinScn() + " maxScn=" + stats2.getMaxScn());
Thread.sleep(2 * 1000);
//restart relay
r3 = new DatabusRelayTestUtil.RelayRunner(relay3);
r3.start();
Thread.sleep(15 * 1000);
r3.pause();
Thread.sleep(35 * 1000);
log.warn("relay3: numDataEvents=" + stats3.getNumDataEvents() + " numWindows=" + stats3.getNumSysEvents() + " minScn=" + stats3.getMinScn() + " maxScn=" + stats3.getMaxScn());
stats2 = relay2.getInboundEventStatisticsCollector().getTotalStats();
log.warn("relay2b: numDataEvents=" + stats2.getNumDataEvents() + " numWindows=" + stats2.getNumSysEvents() + " minScn=" + stats2.getMinScn() + " maxScn=" + stats2.getMaxScn());
log.warn("consumer: " + countingConsumer);
//compare chained relays with 2 gens of tier 0 relays
Assert.assertEquals(stats2.getMinScn(), firstGenMinScn);
Assert.assertEquals(stats2.getMaxScn(), stats3.getMaxScn());
//the total event windows seen by the chained relay will be state of consumption at first failure of relay1 minus 1 overlap window
Assert.assertEquals(stats2.getNumSysEvents(), (firstGenChainWindows - 1) + stats3.getNumSysEvents());
Assert.assertTrue(stats2.getNumDataEvents() > stats3.getNumDataEvents());
//compare source to final client
Assert.assertEquals(countingConsumer.getNumSources(), 2);
Assert.assertEquals(stats2.getNumSysEvents(), countingConsumer.getNumWindows());
boolean sorted = true;
long prev = -1;
log.info(" scn seq on consumer=");
for (Long l : countingConsumer.getEndScns()) {
sorted = sorted && (l >= prev);
prev = l;
log.info(l + " ");
if (!sorted)
break;
}
Assert.assertTrue(sorted);
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2, r3 }, cr);
log.info("end");
}
}
use of com.linkedin.databus.client.DatabusSourcesConnection in project databus by linkedin.
the class TestDatabusRelayMain method testRelayChainingDelayedConsumer.
@Test
public /**
* Chained relay overwrites buffer; consumer starts late; gets only the last n events that fit.
* Makes sure chained relay overwrites buffer;just like main buffer
*/
void testRelayChainingDelayedConsumer() {
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 10;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = 11993;
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1010, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
// create chained relay with only 1 MB buffer
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1;
DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1011, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay2);
resetSCN(relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
// now create client:
String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
String serverName = "localhost:" + chainedRelayPort;
CountingConsumer countingConsumer = new CountingConsumer();
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
cr = new ClientRunner(clientConn);
// async starts for all components;
r1.start();
// start chained relay
r2.start();
// let it run for 10 seconds
Thread.sleep(10 * 1000);
r1.pause();
// start client after the relays have finished
cr.start();
// wait until client got all events or for maxTimeout;
long maxTimeOutMs = 15 * 1000;
long startTime = System.currentTimeMillis();
DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
Thread.sleep(500);
// stats.getSizeDataEvents());
if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
break;
}
}
DbusEventsTotalStats statsChained = relay2.getInboundEventStatisticsCollector().getTotalStats();
LOG.info("Client stats=" + countingConsumer);
LOG.info("Event windows generated=" + stats.getNumSysEvents());
LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
Assert.assertTrue(stats.getNumDataEvents() == statsChained.getNumDataEvents());
Assert.assertTrue(stats.getNumSysEvents() == statsChained.getNumSysEvents());
Assert.assertTrue(stats.getNumDataEvents() > countingConsumer.getNumDataEvents());
Assert.assertTrue(countingConsumer.getNumSources() == 2);
Assert.assertTrue(stats.getNumSysEvents() > countingConsumer.getNumWindows());
} catch (Exception e) {
LOG.error("Exception: " + e);
Assert.assertTrue(false);
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
}
}
use of com.linkedin.databus.client.DatabusSourcesConnection in project databus by linkedin.
the class TestDatabusRelayMain method testRelalyChainingPartialSubscribe.
@Test
public /**
* Client consumes subset of sources from chained relay
*/
void testRelalyChainingPartialSubscribe() {
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 10;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = 11993;
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1006, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
// create chained relay
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1;
DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1007, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay2);
resetSCN(relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
// now create client:
String srcSubscriptionString = "com.linkedin.events.example.Settings";
String serverName = "localhost:" + chainedRelayPort;
CountingConsumer countingConsumer = new CountingConsumer();
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, -1, 30 * 1000, 100, 15 * 1000, 1, true);
cr = new ClientRunner(clientConn);
// start relay r1
r1.start();
// start chained relay
r2.start();
// start client
cr.start();
// Let the relay run for 10s
Thread.sleep(10 * 1000);
r1.pause();
// wait until client got all events or for maxTimeout;
long maxTimeOutMs = 5 * 1000;
long startTime = System.currentTimeMillis();
DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
Thread.sleep(500);
// stats.getSizeDataEvents());
if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
break;
}
}
LOG.info("Client stats=" + countingConsumer);
LOG.info("Event windows generated=" + stats.getNumSysEvents());
LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
Assert.assertTrue(countingConsumer.getNumSources() == 1);
Assert.assertTrue(stats.getNumSysEvents() == countingConsumer.getNumWindows());
Assert.assertTrue(stats.getNumDataEvents() == 2 * countingConsumer.getNumDataEvents());
} catch (Exception e) {
LOG.error("Exception: " + e);
Assert.assertTrue(false);
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
}
}
use of com.linkedin.databus.client.DatabusSourcesConnection in project databus by linkedin.
the class TestDatabusRelayMain method testRelayChainingRestartSCNOffset.
@Test
public /**
* Test relay chaining that utilizes restart SCN offset
*/
void testRelayChainingRestartSCNOffset() throws InterruptedException, InvalidConfigException {
final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayChainingRestartSCNOffset");
log.info("start");
DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null, r3 = null;
ClientRunner cr = null;
try {
String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
// create main relay with random generator
PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
int i = 0;
int eventRatePerSec = 2;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
srcConfigs[i++] = src1;
}
int relayPort = 11993;
DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1016, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay1);
r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
// create chained relay
PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
int j = 0;
for (String[] srcs : srcNames) {
PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
chainedSrcConfigs[j++] = src1;
}
int chainedRelayPort = relayPort + 1;
DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1017, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
Assert.assertTrue(null != relay2);
r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
resetSCN(relay2);
// now create client:
String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
String serverName = "localhost:" + chainedRelayPort;
final CountingConsumer countingConsumer = new CountingConsumer();
DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
cr = new ClientRunner(clientConn);
// async starts for all components;
r1.start();
// start chained relay
r2.start();
// start client
Thread.sleep(500);
cr.start();
r1.pause();
// let it run for 10 seconds
Thread.sleep(10 * 1000);
// wait until client got all events or for maxTimeout;
final long maxTimeOutMs = 5 * 1000;
final DbusEventsTotalStats dbRelayStats = relay1.getInboundEventStatisticsCollector().getTotalStats();
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return countingConsumer.getNumWindows() == dbRelayStats.getNumSysEvents();
}
}, "consumer caught up", maxTimeOutMs, log);
log.info("Client stats=" + countingConsumer);
log.info("Event windows generated=" + dbRelayStats.getNumSysEvents());
log.info("numDataEvents=" + dbRelayStats.getNumDataEvents() + " numWindows=" + dbRelayStats.getNumSysEvents() + " size=" + dbRelayStats.getSizeDataEvents());
Assert.assertEquals(dbRelayStats.getNumDataEvents(), countingConsumer.getNumDataEvents());
Assert.assertEquals(countingConsumer.getNumSources(), 2);
Assert.assertEquals(dbRelayStats.getNumSysEvents(), countingConsumer.getNumWindows());
cr.shutdown();
boolean s2 = r2.shutdown(2000);
Assert.assertTrue(s2);
Assert.assertTrue(!r2.isAlive());
//start r3; new chained relay with restart SCN offset; we get some data;
chainedSrcConfigs[0].setRestartScnOffset(dbRelayStats.getMaxScn() - dbRelayStats.getPrevScn());
DatabusRelayMain relay3 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1018, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
r3 = new DatabusRelayTestUtil.RelayRunner(relay3);
r3.start();
final DbusEventsTotalStats newChainedRlyStats = relay3.getInboundEventStatisticsCollector().getTotalStats();
TestUtil.assertWithBackoff(new ConditionCheck() {
@Override
public boolean check() {
return newChainedRlyStats.getNumDataEvents() > 0;
}
}, "new chained relay running", 5000, log);
log.info("Stats3= numDataEvents=" + newChainedRlyStats.getNumDataEvents() + " numWindows=" + newChainedRlyStats.getNumSysEvents() + " size=" + newChainedRlyStats.getSizeDataEvents());
} finally {
cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2, r3 }, cr);
log.info("end");
}
}
Aggregations